diff --git a/.gitmodules b/.gitmodules index 2b085e15059..f3653b475c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -51,9 +51,6 @@ [submodule "third_party/protoc-gen-validate"] path = third_party/protoc-gen-validate url = https://github.com/envoyproxy/protoc-gen-validate.git -[submodule "third_party/upb"] - path = third_party/upb - url = https://github.com/protocolbuffers/upb.git [submodule "third_party/udpa"] path = third_party/udpa url = https://github.com/cncf/udpa.git diff --git a/third_party/upb b/third_party/upb deleted file mode 160000 index 931bbecbd32..00000000000 --- a/third_party/upb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 931bbecbd3230ae7f22efa5d203639facc47f719 diff --git a/third_party/upb/.bazelci/presubmit.yml b/third_party/upb/.bazelci/presubmit.yml new file mode 100644 index 00000000000..38ca79304f9 --- /dev/null +++ b/third_party/upb/.bazelci/presubmit.yml @@ -0,0 +1,10 @@ +--- +tasks: + ubuntu: + platform: ubuntu1604 + test_targets: + - //... + macos: + platform: macos + test_targets: + - //... diff --git a/third_party/upb/.gitignore b/third_party/upb/.gitignore new file mode 100644 index 00000000000..c9bd5becf81 --- /dev/null +++ b/third_party/upb/.gitignore @@ -0,0 +1,4 @@ +*.s?? +obj/ +lib/ +bazel-* diff --git a/third_party/upb/.gitmodules b/third_party/upb/.gitmodules new file mode 100644 index 00000000000..8b52c1d2165 --- /dev/null +++ b/third_party/upb/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/protobuf"] + path = third_party/protobuf + url = https://github.com/google/protobuf.git diff --git a/third_party/upb/BUILD b/third_party/upb/BUILD new file mode 100644 index 00000000000..ad85b202a9a --- /dev/null +++ b/third_party/upb/BUILD @@ -0,0 +1,727 @@ +load( + "//bazel:build_defs.bzl", + "generated_file_staleness_test", + "licenses", # copybara:strip_for_google3 + "lua_binary", + "lua_cclibrary", + "lua_library", + "lua_test", + "make_shell_script", + "upb_amalgamation", +) +load( + "//bazel:upb_proto_library.bzl", + "upb_proto_library", + "upb_proto_reflection_library", +) + +licenses(["notice"]) # BSD (Google-authored w/ possible external contributions) + +exports_files([ + "LICENSE", + "build_defs", +]) + +CPPOPTS = [ + # copybara:strip_for_google3_begin + "-Werror", + "-Wno-long-long", + # copybara:strip_end +] + +COPTS = CPPOPTS + [ + # copybara:strip_for_google3_begin + "-pedantic", + "-Wstrict-prototypes", + # copybara:strip_end +] + +config_setting( + name = "darwin", + values = {"cpu": "darwin"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "windows", + constraint_values = ["@bazel_tools//platforms:windows"], +) + +config_setting( + name = "fuzz", + values = {"define": "fuzz=true"}, +) + +# Public C/C++ libraries ####################################################### + +cc_library( + name = "upb", + srcs = [ + "upb/decode.c", + "upb/encode.c", + "upb/generated_util.h", + "upb/msg.c", + "upb/msg.h", + "upb/port.c", + "upb/port_def.inc", + "upb/port_undef.inc", + "upb/table.c", + "upb/table.int.h", + "upb/upb.c", + ], + hdrs = [ + "upb/decode.h", + "upb/encode.h", + "upb/upb.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + visibility = ["//visibility:public"], +) + +# Common support routines used by generated code. This library has no +# implementation, but depends on :upb and exposes a few more hdrs. +# +# This is public only because we have no way of visibility-limiting it to +# upb_proto_library() only. This interface is not stable and by using it you +# give up any backward compatibility guarantees. +cc_library( + name = "generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + hdrs = [ + "upb/generated_util.h", + "upb/msg.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + textual_hdrs = [ + "upb/port_def.inc", + "upb/port_undef.inc", + ], + visibility = ["//visibility:public"], + deps = [":upb"], +) + +upb_proto_library( + name = "descriptor_upbproto", + visibility = ["//visibility:public"], + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +cc_library( + name = "reflection", + srcs = [ + "upb/def.c", + "upb/msgfactory.c", + ], + hdrs = [ + "upb/def.h", + "upb/msgfactory.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + visibility = ["//visibility:public"], + deps = [ + ":descriptor_upbproto", + ":table", + ":upb", + ], +) + +# Internal C/C++ libraries ##################################################### + +cc_library( + name = "table", + hdrs = ["upb/table.int.h"], + deps = [":upb"], +) + +# Legacy C/C++ Libraries (not recommended for new code) ######################## + +cc_library( + name = "legacy_msg_reflection", + srcs = [ + "upb/legacy_msg_reflection.c", + ], + hdrs = ["upb/legacy_msg_reflection.h"], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":table", + ":upb", + ], +) + +cc_library( + name = "handlers", + srcs = [ + "upb/handlers.c", + "upb/handlers-inl.h", + "upb/sink.c", + ], + hdrs = [ + "upb/handlers.h", + "upb/sink.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":reflection", + ":table", + ":upb", + ], +) + +cc_library( + name = "upb_pb", + srcs = [ + "upb/pb/compile_decoder.c", + "upb/pb/decoder.c", + "upb/pb/decoder.int.h", + "upb/pb/encoder.c", + "upb/pb/textprinter.c", + "upb/pb/varint.c", + "upb/pb/varint.int.h", + ], + hdrs = [ + "upb/pb/decoder.h", + "upb/pb/encoder.h", + "upb/pb/textprinter.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":descriptor_upbproto", + ":handlers", + ":reflection", + ":table", + ":upb", + ], +) + +# copybara:strip_for_google3_begin +cc_library( + name = "upb_json", + srcs = [ + "upb/json/parser.c", + "upb/json/printer.c", + ], + hdrs = [ + "upb/json/parser.h", + "upb/json/printer.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":upb", + ":upb_pb", + ], +) +# copybara:strip_end + +cc_library( + name = "upb_cc_bindings", + hdrs = [ + "upb/bindings/stdc++/string.h", + ], + deps = [ + ":descriptor_upbproto", + ":handlers", + ":upb", + ], +) + +# upb compiler ################################################################# + +cc_library( + name = "upbc_generator", + srcs = [ + "upbc/generator.cc", + "upbc/message_layout.cc", + "upbc/message_layout.h", + ], + hdrs = ["upbc/generator.h"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + "@absl//absl/base:core_headers", + "@absl//absl/container:flat_hash_map", + "@absl//absl/strings", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:protoc_lib", + ], +) + +cc_binary( + name = "protoc-gen-upb", + srcs = ["upbc/main.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + visibility = ["//visibility:public"], + deps = [ + ":upbc_generator", + "@com_google_protobuf//:protoc_lib", + ], +) + +# We strip the tests and remaining rules from google3 until the upb_proto_library() +# and upb_proto_reflection_library() rules are fixed. + +# C/C++ tests ################################################################## + +cc_binary( + name = "benchmark", + testonly = 1, + srcs = ["tests/benchmark.cc"], + deps = [ + ":descriptor_upbproto", + ":descriptor_upbreflection", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "upb_test", + testonly = 1, + srcs = [ + "tests/testmain.cc", + ], + hdrs = [ + "tests/test_util.h", + "tests/upb_test.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":handlers", + ":upb", + ], +) + +cc_test( + name = "test_varint", + srcs = [ + "tests/pb/test_varint.c", + "upb/pb/varint.int.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":upb", + ":upb_pb", + ":upb_test", + ], +) + +proto_library( + name = "test_decoder_proto", + srcs = [ + "tests/pb/test_decoder.proto", + ], +) + +upb_proto_reflection_library( + name = "test_decoder_upbproto", + deps = [":test_decoder_proto"], +) + +cc_test( + name = "test_decoder", + srcs = [ + "tests/pb/test_decoder.cc", + "upb/pb/varint.int.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":handlers", + ":test_decoder_upbproto", + ":upb", + ":upb_pb", + ":upb_test", + ], +) + +proto_library( + name = "test_cpp_proto", + srcs = [ + "tests/test_cpp.proto", + ], +) + +upb_proto_reflection_library( + name = "test_cpp_upbproto", + deps = ["test_cpp_proto"], +) + +cc_test( + name = "test_cpp", + srcs = ["tests/test_cpp.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":handlers", + ":reflection", + ":test_cpp_upbproto", + ":upb", + ":upb_pb", + ":upb_test", + ], +) + +cc_test( + name = "test_table", + srcs = ["tests/test_table.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":table", + ":upb", + ":upb_test", + ], +) + +# OSS-Fuzz test +cc_binary( + name = "file_descriptor_parsenew_fuzzer", + testonly = 1, + srcs = ["tests/file_descriptor_parsenew_fuzzer.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }) + select({ + "//conditions:default": [], + ":fuzz": ["-fsanitize=fuzzer,address"], + }), + defines = select({ + "//conditions:default": [], + ":fuzz": ["HAVE_FUZZER"], + }), + deps = [ + ":descriptor_upbproto", + ":upb", + ], +) + +# copybara:strip_for_google3_begin +upb_proto_reflection_library( + name = "descriptor_upbreflection", + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +cc_test( + name = "test_encoder", + srcs = ["tests/pb/test_encoder.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":descriptor_upbproto", + ":descriptor_upbreflection", + ":upb", + ":upb_cc_bindings", + ":upb_pb", + ":upb_test", + ], +) + +proto_library( + name = "test_json_enum_from_separate", + srcs = ["tests/json/enum_from_separate_file.proto"], + deps = [":test_json_proto"], +) + +proto_library( + name = "test_json_proto", + srcs = ["tests/json/test.proto"], +) + +upb_proto_reflection_library( + name = "test_json_upbprotoreflection", + deps = ["test_json_proto"], +) + +upb_proto_library( + name = "test_json_enum_from_separate_upbproto", + deps = [":test_json_enum_from_separate"], +) + +upb_proto_library( + name = "test_json_upbproto", + deps = [":test_json_proto"], +) + +cc_test( + name = "test_json", + srcs = [ + "tests/json/test_json.cc", + ], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":test_json_upbproto", + ":test_json_upbprotoreflection", + ":upb_json", + ":upb_test", + ], +) +# copybara:strip_end + +upb_proto_library( + name = "conformance_proto_upb", + testonly = 1, + deps = ["@com_google_protobuf//:conformance_proto"], +) + +upb_proto_library( + name = "test_messages_proto3_proto_upb", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto3_proto"], +) + +cc_binary( + name = "conformance_upb", + testonly = 1, + srcs = [ + "tests/conformance_upb.c", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }) + ["-Ibazel-out/k8-fastbuild/bin"], + deps = [ + ":conformance_proto_upb", + ":test_messages_proto3_proto_upb", + ":upb", + ], +) + +make_shell_script( + name = "gen_test_conformance_upb", + out = "test_conformance_upb.sh", + contents = "external/com_google_protobuf/conformance_test_runner ./conformance_upb", +) + +sh_test( + name = "test_conformance_upb", + srcs = ["test_conformance_upb.sh"], + data = [ + "tests/conformance_upb_failures.txt", + ":conformance_upb", + "@com_google_protobuf//:conformance_test_runner", + ], +) + +# copybara:strip_for_google3_begin + +# Amalgamation ################################################################# + +py_binary( + name = "amalgamate", + srcs = ["tools/amalgamate.py"], +) + +upb_amalgamation( + name = "gen_amalgamation", + outs = [ + "upb.c", + "upb.h", + ], + amalgamator = ":amalgamate", + libs = [ + ":upb", + ":descriptor_upbproto", + ":reflection", + ":handlers", + ":upb_pb", + ":upb_json", + ], +) + +cc_library( + name = "amalgamation", + srcs = ["upb.c"], + hdrs = ["upb.h"], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), +) + +# Lua libraries. ############################################################### + +lua_cclibrary( + name = "lua/upb_c", + srcs = [ + "upb/bindings/lua/def.c", + "upb/bindings/lua/msg.c", + "upb/bindings/lua/upb.c", + ], + hdrs = [ + "upb/bindings/lua/upb.h", + ], + deps = [ + "legacy_msg_reflection", + "upb", + "upb_pb", + ], +) + +lua_library( + name = "lua/upb", + srcs = ["upb/bindings/lua/upb.lua"], + luadeps = ["lua/upb_c"], + strip_prefix = "upb/bindings/lua", +) + +lua_cclibrary( + name = "lua/upb/pb_c", + srcs = ["upb/bindings/lua/upb/pb.c"], + luadeps = ["lua/upb_c"], + deps = ["upb_pb"], +) + +lua_library( + name = "lua/upb/pb", + srcs = ["upb/bindings/lua/upb/pb.lua"], + luadeps = [ + "lua/upb", + "lua/upb/pb_c", + ], + strip_prefix = "upb/bindings/lua", +) + +# Lua tests. ################################################################### + +lua_test( + name = "lua/test_upb", + luadeps = ["lua/upb"], + luamain = "tests/bindings/lua/test_upb.lua", +) + +lua_test( + name = "lua/test_upb_pb", + luadeps = ["lua/upb/pb"], + luamain = "tests/bindings/lua/test_upb.pb.lua", +) + +# Test the CMake build ######################################################### + +filegroup( + name = "cmake_files", + srcs = glob([ + "CMakeLists.txt", + "generated_for_cmake/**/*", + "google/**/*", + "upbc/**/*", + "upb/**/*", + "tests/**/*", + ]), +) + +make_shell_script( + name = "gen_run_cmake_build", + out = "run_cmake_build.sh", + contents = "find . && mkdir build && cd build && cmake .. && make -j8 && make test", +) + +sh_test( + name = "cmake_build", + srcs = ["run_cmake_build.sh"], + data = [":cmake_files"], +) + +# Generated files ############################################################## + +exports_files(["tools/staleness_test.py"]) + +py_library( + name = "staleness_test_lib", + testonly = 1, + srcs = ["tools/staleness_test_lib.py"], +) + +py_binary( + name = "make_cmakelists", + srcs = ["tools/make_cmakelists.py"], +) + +genrule( + name = "gen_cmakelists", + srcs = [ + "BUILD", + "WORKSPACE", + ":cmake_files", + ], + outs = ["generated-in/CMakeLists.txt"], + cmd = "$(location :make_cmakelists) $@", + tools = [":make_cmakelists"], +) + +genrule( + name = "generate_json_ragel", + srcs = ["upb/json/parser.rl"], + outs = ["upb/json/parser.c"], + cmd = "$(location @ragel//:ragelc) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", + tools = ["@ragel//:ragelc"], +) + +genrule( + name = "copy_json_ragel", + srcs = ["upb/json/parser.c"], + outs = ["generated-in/generated_for_cmake/upb/json/parser.c"], + cmd = "cp $< $@", +) + +genrule( + name = "copy_protos", + srcs = [":descriptor_upbproto"], + outs = [ + "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.c", + "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.h", + ], + cmd = "cp $(SRCS) $(@D)/generated-in/generated_for_cmake/google/protobuf", +) + +generated_file_staleness_test( + name = "test_generated_files", + outs = [ + "CMakeLists.txt", + "generated_for_cmake/google/protobuf/descriptor.upb.c", + "generated_for_cmake/google/protobuf/descriptor.upb.h", + "generated_for_cmake/upb/json/parser.c", + ], + generated_pattern = "generated-in/%s", +) + +# copybara:strip_end diff --git a/third_party/upb/CMakeLists.txt b/third_party/upb/CMakeLists.txt new file mode 100644 index 00000000000..836c5ff1fe5 --- /dev/null +++ b/third_party/upb/CMakeLists.txt @@ -0,0 +1,147 @@ +# This file was generated from BUILD using tools/make_cmakelists.py. + +cmake_minimum_required(VERSION 3.1) + +if(${CMAKE_VERSION} VERSION_LESS 3.12) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.12) +endif() + +cmake_minimum_required (VERSION 3.0) +cmake_policy(SET CMP0048 NEW) + +project(upb) + + +# Prevent CMake from setting -rdynamic on Linux (!!). +SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set default build type. +if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() + +# When using Ninja, compiler output won't be colorized without this. +include(CheckCXXCompilerFlag) +CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS) +if(SUPPORTS_COLOR_ALWAYS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") +endif() + +# Implement ASAN/UBSAN options +if(UPB_ENABLE_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") +endif() + +if(UPB_ENABLE_UBSAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") +endif() + +include_directories(.) +include_directories(generated_for_cmake) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +if(APPLE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") +elseif(UNIX) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") +endif() + +enable_testing() + +add_library(upb + upb/decode.c + upb/encode.c + upb/generated_util.h + upb/msg.c + upb/msg.h + upb/port.c + upb/port_def.inc + upb/port_undef.inc + upb/table.c + upb/table.int.h + upb/upb.c + upb/decode.h + upb/encode.h + upb/upb.h) +add_library(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE) +target_link_libraries(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE + upb) +add_library(reflection + upb/def.c + upb/msgfactory.c + upb/def.h + upb/msgfactory.h) +target_link_libraries(reflection + descriptor_upbproto + table + upb) +add_library(table INTERFACE) +target_link_libraries(table INTERFACE + upb) +add_library(legacy_msg_reflection + upb/legacy_msg_reflection.c + upb/legacy_msg_reflection.h) +target_link_libraries(legacy_msg_reflection + table + upb) +add_library(handlers + upb/handlers.c + upb/handlers-inl.h + upb/sink.c + upb/handlers.h + upb/sink.h) +target_link_libraries(handlers + reflection + table + upb) +add_library(upb_pb + upb/pb/compile_decoder.c + upb/pb/decoder.c + upb/pb/decoder.int.h + upb/pb/encoder.c + upb/pb/textprinter.c + upb/pb/varint.c + upb/pb/varint.int.h + upb/pb/decoder.h + upb/pb/encoder.h + upb/pb/textprinter.h) +target_link_libraries(upb_pb + descriptor_upbproto + handlers + reflection + table + upb) +add_library(upb_json + generated_for_cmake/upb/json/parser.c + upb/json/printer.c + upb/json/parser.h + upb/json/printer.h) +target_link_libraries(upb_json + upb + upb_pb) +add_library(upb_cc_bindings INTERFACE) +target_link_libraries(upb_cc_bindings INTERFACE + descriptor_upbproto + handlers + upb) +add_library(upb_test + tests/testmain.cc + tests/test_util.h + tests/upb_test.h) +target_link_libraries(upb_test + handlers + upb) + + diff --git a/third_party/upb/CONTRIBUTING.md b/third_party/upb/CONTRIBUTING.md new file mode 100644 index 00000000000..2f866b4e291 --- /dev/null +++ b/third_party/upb/CONTRIBUTING.md @@ -0,0 +1,7 @@ +## Signing the CLA + +Please sign the [Google Contributor License Agreement +(CLA)](https://cla.developers.google.com/) +before sending pull requests. For any code changes to be +accepted, the CLA must be signed. It's a quick process, I +promise! diff --git a/third_party/upb/DESIGN.md b/third_party/upb/DESIGN.md new file mode 100644 index 00000000000..a7a8a284516 --- /dev/null +++ b/third_party/upb/DESIGN.md @@ -0,0 +1,72 @@ + +μpb Design +---------- + +μpb has the following design goals: + +- C89 compatible. +- small code size (both for the core library and generated messages). +- fast performance (hundreds of MB/s). +- idiomatic for C programs. +- easy to wrap in high-level languages (Python, Ruby, Lua, etc) with + good performance and all standard protobuf features. +- hands-off about memory management, allowing for easy integration + with existing VMs and/or garbage collectors. +- offers binary ABI compatibility between apps, generated messages, and + the core library (doesn't require re-generating messages or recompiling + your application when the core library changes). +- provides all features that users expect from a protobuf library + (generated messages in C, reflection, text format, etc.). +- layered, so the core is small and doesn't require descriptors. +- tidy about symbol references, so that any messages or features that + aren't used by a C program can have their code GC'd by the linker. +- possible to use protobuf binary format without leaking message/field + names into the binary. + +μpb accomplishes these goals by keeping a very small core that does not contain +descriptors. We need some way of knowing what fields are in each message and +where they live, but instead of descriptors, we keep a small/lightweight summary +of the .proto file. We call this a `upb_msglayout`. It contains the bare +minimum of what we need to know to parse and serialize protobuf binary format +into our internal representation for messages, `upb_msg`. + +The core then contains functions to parse/serialize a message, given a `upb_msg*` +and a `const upb_msglayout*`. + +This approach is similar to [nanopb](https://github.com/nanopb/nanopb) which +also compiles message definitions to a compact, internal representation without +names. However nanopb does not aim to be a fully-featured library, and has no +support for text format, JSON, or descriptors. μpb is unique in that it has a +small core similar to nanopb (though not quite as small), but also offers a +full-featured protobuf library for applications that want reflection, text +format, JSON format, etc. + +Without descriptors, the core doesn't have access to field names, so it cannot +parse/serialize to protobuf text format or JSON. Instead this functionality +lives in separate modules that depend on the module implementing descriptors. +With the descriptor module we can parse/serialize binary descriptors and +validate that they follow all the rules of protobuf schemas. + +To provide binary compatibility, we version the structs that generated messages +use to create a `upb_msglayout*`. The current initializers are +`upb_msglayout_msginit_v1`, `upb_msglayout_fieldinit_v1`, etc. Then +`upb_msglayout*` uses these as its internal representation. If upb changes its +internal representation for a `upb_msglayout*`, it will also include code to +convert the old representation to the new representation. This will use some +more memory/CPU at runtime to convert between the two, but apps that statically +link μpb will never need to worry about this. + +TODO +---- + +1. revise our generated code until it is in a state where we feel comfortable + committing to API/ABI stability for it. In particular there is an open + question of whether non-ABI-compatible field accesses should have a + fastpath different from the ABI-compatible field access. +1. Add missing features (maps, extensions, unknown fields). +1. Flesh out C++ wrappers. +1. *(lower-priority)*: revise all of the existing encoders/decoders and + handlers. We probably will want to keep handlers, since they let us decouple + encoders/decoders from `upb_msg`, but we need to simplify all of that a LOT. + Likely we will want to make handlers only per-message instead of per-field, + except for variable-length fields. diff --git a/third_party/upb/LICENSE b/third_party/upb/LICENSE new file mode 100644 index 00000000000..da939845db6 --- /dev/null +++ b/third_party/upb/LICENSE @@ -0,0 +1,26 @@ + +Copyright (c) 2009-2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of any other + contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY GOOGLE INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL GOOGLE INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/upb/README.md b/third_party/upb/README.md new file mode 100644 index 00000000000..a08aa0e9092 --- /dev/null +++ b/third_party/upb/README.md @@ -0,0 +1,134 @@ + +# μpb - a small protobuf implementation in C + +|Platform|Build Status| +|--------|------------| +|macOS|[![Build Status](https://storage.googleapis.com/upb-kokoro-results/status-badge/macos.png)](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Aupb%2Fmacos%2Fcontinuous)| +|ubuntu|[![Build Status](https://storage.googleapis.com/upb-kokoro-results/status-badge/ubuntu.png)](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Aupb%2Fubuntu%2Fcontinuous)| + +μpb (often written 'upb') is a small protobuf implementation written in C. + +upb generates a C API for creating, parsing, and serializing messages +as declared in `.proto` files. upb is heavily arena-based: all +messages always live in an arena (note: the arena can live in stack or +static memory if desired). Here is a simple example: + +```c +#include "conformance/conformance.upb.h" + +void foo(const char* data, size_t size) { + upb_arena *arena; + + /* Generated message type. */ + conformance_ConformanceRequest *request; + conformance_ConformanceResponse *response; + + arena = upb_arena_new(); + request = conformance_ConformanceRequest_parse(data, size, arena); + response = conformance_ConformanceResponse_new(arena); + + switch (conformance_ConformanceRequest_payload_case(request)) { + case conformance_ConformanceRequest_payload_protobuf_payload: { + upb_strview payload = conformance_ConformanceRequest_protobuf_payload(request); + // ... + break; + } + + case conformance_ConformanceRequest_payload_NOT_SET: + fprintf(stderr, "conformance_upb: Request didn't have payload.\n"); + break; + + default: { + static const char msg[] = "Unsupported input format."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + break; + } + } + + /* Frees all messages on the arena. */ + upb_arena_free(arena); +} +``` + +API and ABI are both subject to change! Please do not distribute +as a shared library for this reason (for now at least). + +## Using upb in your project + +Currently only Bazel is supported (CMake support is partial and incomplete +but full CMake support is an eventual goal). + +To use upb in your Bazel project, first add upb to your `WORKSPACE` file, +either as a `git_repository()` or as a `new_local_repository()` with a +Git Submodule. (For an example, see `examples/bazel/ in this repo). + +```python +# Add this to your WORKSPACE file. +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "upb", + remote = "https://github.com/protocolbuffers/upb.git", + commit = "d16bf99ac4658793748cda3251226059892b3b7b", +) + +load("@upb//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() +``` + +Then in your BUILD file you can add `upb_proto_library()` rules that +generate code for a corresponding `proto_library()` rule. For +example: + +```python +# Add this to your BUILD file. +load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library") + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) + +upb_proto_library( + name = "foo_upbproto", + deps = [":foo_proto"], +) + +cc_binary( + name = "test_binary", + srcs = ["test_binary.c"], + deps = [":foo_upbproto"], +) +``` + +Then in your `.c` file you can #include the generated header: + +```c +#include "foo.upb.h" + +/* Insert code that uses generated types. */ +``` + +## Old "handlers" interfaces + +This library contains several semi-deprecated interfaces (see BUILD +file for more info about which interfaces are deprecated). These +deprecated interfaces are still used in some significant projects, +such as the Ruby and PHP C bindings for protobuf in the [main protobuf +repo](https://github.com/protocolbuffers/protobuf). The goal is to +migrate the Ruby/PHP bindings to use the newer, simpler interfaces +instead. Please do not use the old interfaces in new code. + +## Lua bindings + +This repo has some Lua bindings for the core library. These are +experimental and very incomplete. These are currently included in +order to validate that the C API is suitable for wrapping. As the +project matures these Lua bindings may become publicly available. + +## Contact + +Author: Josh Haberman ([jhaberman@gmail.com](mailto:jhaberman@gmail.com), +[haberman@google.com](mailto:haberman@google.com)) diff --git a/third_party/upb/WORKSPACE b/third_party/upb/WORKSPACE new file mode 100644 index 00000000000..c1c8c3be9b7 --- /dev/null +++ b/third_party/upb/WORKSPACE @@ -0,0 +1,39 @@ +workspace(name = "upb") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() + +http_archive( + name = "lua", + build_file = "//bazel:lua.BUILD", + sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b", + strip_prefix = "lua-5.2.4", + urls = [ + "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz", + "https://www.lua.org/ftp/lua-5.2.4.tar.gz", + ], +) + +http_archive( + name = "ragel", + build_file = "//bazel:ragel.BUILD", + sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", + strip_prefix = "ragel-6.10", + urls = ["http://www.colm.net/files/ragel/ragel-6.10.tar.gz"], +) + +http_archive( + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip"], # 2019-01-07 + strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb", + sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86", +) + +http_archive( + name = "com_github_google_benchmark", + urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"], + strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be", + sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3", +) diff --git a/third_party/upb/bazel/BUILD b/third_party/upb/bazel/BUILD new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/upb/bazel/build_defs.bzl b/third_party/upb/bazel/build_defs.bzl new file mode 100644 index 00000000000..08bb44ee3f7 --- /dev/null +++ b/third_party/upb/bazel/build_defs.bzl @@ -0,0 +1,221 @@ +"""Internal rules for building upb.""" + +load(":upb_proto_library.bzl", "GeneratedSrcsInfo") + +def _librule(name): + return name + "_lib" + +def _get_real_short_path(file): + # For some reason, files from other archives have short paths that look like: + # ../com_google_protobuf/google/protobuf/descriptor.proto + short_path = file.short_path + if short_path.startswith("../"): + second_slash = short_path.index("/", 3) + short_path = short_path[second_slash + 1:] + return short_path + +def _get_real_root(file): + real_short_path = _get_real_short_path(file) + return file.path[:-len(real_short_path) - 1] + +def _get_real_roots(files): + roots = {} + for file in files: + real_root = _get_real_root(file) + if real_root: + roots[real_root] = True + return roots.keys() + +def lua_cclibrary(name, srcs, hdrs = [], deps = [], luadeps = []): + lib_rule = name + "_lib" + so_rule = "lib" + name + ".so" + so_file = _remove_prefix(name, "lua/") + ".so" + + native.cc_library( + name = _librule(name), + hdrs = hdrs, + srcs = srcs, + deps = deps + [_librule(dep) for dep in luadeps] + ["@lua//:liblua_headers"], + ) + + native.cc_binary( + name = so_rule, + linkshared = True, + deps = [_librule(name)], + linkopts = select({ + ":darwin": [ + "-undefined dynamic_lookup", + ], + "//conditions:default": [], + }), + ) + + native.genrule( + name = name + "_copy", + srcs = [":" + so_rule], + outs = [so_file], + cmd = "cp $< $@", + ) + + native.filegroup( + name = name, + data = [so_file], + ) + +def _remove_prefix(str, prefix): + if not str.startswith(prefix): + fail("%s doesn't start with %s" % (str, prefix)) + return str[len(prefix):] + +def _remove_suffix(str, suffix): + if not str.endswith(suffix): + fail("%s doesn't end with %s" % (str, suffix)) + return str[:-len(suffix)] + +def lua_library(name, srcs, strip_prefix, luadeps = []): + outs = [_remove_prefix(src, strip_prefix + "/") for src in srcs] + native.genrule( + name = name + "_copy", + srcs = srcs, + outs = outs, + cmd = "cp $(SRCS) $(@D)", + ) + + native.filegroup( + name = name, + data = outs + luadeps, + ) + +def make_shell_script(name, contents, out): + contents = contents.replace("$", "$$") + native.genrule( + name = "gen_" + name, + outs = [out], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % contents, + ) + +def _lua_binary_or_test(name, luamain, luadeps, rule): + script = name + ".sh" + + make_shell_script( + name = "gen_" + name, + out = script, + contents = """ +BASE=$(dirname $(rlocation upb/upb_c.so)) +export LUA_CPATH="$BASE/?.so" +export LUA_PATH="$BASE/?.lua" +$(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" +""", + ) + + rule( + name = name, + srcs = [script], + data = ["@lua//:lua", luamain] + luadeps, + ) + +def lua_binary(name, luamain, luadeps = []): + _lua_binary_or_test(name, luamain, luadeps, native.sh_binary) + +def lua_test(name, luamain, luadeps = []): + _lua_binary_or_test(name, luamain, luadeps, native.sh_test) + +def generated_file_staleness_test(name, outs, generated_pattern): + """Tests that checked-in file(s) match the contents of generated file(s). + + The resulting test will verify that all output files exist and have the + correct contents. If the test fails, it can be invoked with --fix to + bring the checked-in files up to date. + + Args: + name: Name of the rule. + outs: the checked-in files that are copied from generated files. + generated_pattern: the pattern for transforming each "out" file into a + generated file. For example, if generated_pattern="generated/%s" then + a file foo.txt will look for generated file generated/foo.txt. + """ + + script_name = name + ".py" + script_src = "//:tools/staleness_test.py" + + # Filter out non-existing rules so Blaze doesn't error out before we even + # run the test. + existing_outs = native.glob(include = outs) + + # The file list contains a few extra bits of information at the end. + # These get unpacked by the Config class in staleness_test_lib.py. + file_list = outs + [generated_pattern, native.package_name() or ".", name] + + native.genrule( + name = name + "_makescript", + outs = [script_name], + srcs = [script_src], + testonly = 1, + cmd = "cat $(location " + script_src + ") > $@; " + + "sed -i.bak -e 's|INSERT_FILE_LIST_HERE|" + "\\\n ".join(file_list) + "|' $@", + ) + + native.py_test( + name = name, + srcs = [script_name], + data = existing_outs + [generated_pattern % file for file in outs], + deps = [ + "//:staleness_test_lib", + ], + ) + +# upb_amalgamation() rule, with file_list aspect. + +SrcList = provider( + fields = { + "srcs": "list of srcs", + }, +) + +def _file_list_aspect_impl(target, ctx): + if GeneratedSrcsInfo in target: + srcs = target[GeneratedSrcsInfo] + return [SrcList(srcs = srcs.srcs + srcs.hdrs)] + + srcs = [] + for src in ctx.rule.attr.srcs: + srcs += src.files.to_list() + for hdr in ctx.rule.attr.hdrs: + srcs += hdr.files.to_list() + for hdr in ctx.rule.attr.textual_hdrs: + srcs += hdr.files.to_list() + return [SrcList(srcs = srcs)] + +_file_list_aspect = aspect( + implementation = _file_list_aspect_impl, +) + +def _upb_amalgamation(ctx): + inputs = [] + for lib in ctx.attr.libs: + inputs += lib[SrcList].srcs + srcs = [src for src in inputs if src.path.endswith("c")] + ctx.actions.run( + inputs = inputs, + outputs = ctx.outputs.outs, + arguments = [ctx.bin_dir.path + "/"] + [f.path for f in srcs] + ["-I" + root for root in _get_real_roots(inputs)], + progress_message = "Making amalgamation", + executable = ctx.executable.amalgamator, + ) + return [] + +upb_amalgamation = rule( + attrs = { + "amalgamator": attr.label( + executable = True, + cfg = "host", + ), + "libs": attr.label_list(aspects = [_file_list_aspect]), + "outs": attr.output_list(), + }, + implementation = _upb_amalgamation, +) + +def licenses(*args): + # No-op (for Google-internal usage). + pass diff --git a/third_party/upb/bazel/lua.BUILD b/third_party/upb/bazel/lua.BUILD new file mode 100644 index 00000000000..7be0b594e56 --- /dev/null +++ b/third_party/upb/bazel/lua.BUILD @@ -0,0 +1,102 @@ +package( + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "liblua_headers", + defines = ["LUA_USE_LINUX"], + hdrs = [ + "src/lauxlib.h", + "src/lua.h", + "src/lua.hpp", + "src/luaconf.h", + "src/lualib.h", + ], + includes = ["src"], +) + +cc_library( + name = "liblua", + srcs = [ + "src/lapi.c", + "src/lapi.h", + "src/lauxlib.c", + "src/lauxlib.h", + "src/lbaselib.c", + "src/lbitlib.c", + "src/lcode.c", + "src/lcode.h", + "src/lcorolib.c", + "src/lctype.c", + "src/lctype.h", + "src/ldblib.c", + "src/ldebug.c", + "src/ldebug.h", + "src/ldo.c", + "src/ldo.h", + "src/ldump.c", + "src/lfunc.c", + "src/lfunc.h", + "src/lgc.c", + "src/lgc.h", + "src/linit.c", + "src/liolib.c", + "src/llex.c", + "src/llex.h", + "src/llimits.h", + "src/lmathlib.c", + "src/lmem.c", + "src/lmem.h", + "src/loadlib.c", + "src/lobject.c", + "src/lobject.h", + "src/lopcodes.c", + "src/lopcodes.h", + "src/loslib.c", + "src/lparser.c", + "src/lparser.h", + "src/lstate.c", + "src/lstate.h", + "src/lstring.c", + "src/lstring.h", + "src/lstrlib.c", + "src/ltable.c", + "src/ltable.h", + "src/ltablib.c", + "src/ltm.c", + "src/ltm.h", + "src/lundump.c", + "src/lundump.h", + "src/lvm.c", + "src/lvm.h", + "src/lzio.c", + "src/lzio.h", + ], + defines = ["LUA_USE_LINUX"], + hdrs = [ + "src/lauxlib.h", + "src/lua.h", + "src/lua.hpp", + "src/luaconf.h", + "src/lualib.h", + ], + includes = ["src"], + linkopts = [ + "-lm", + "-ldl", + ], +) + +cc_binary( + name = "lua", + srcs = [ + "src/lua.c", + ], + deps = [ + ":liblua", + ], + linkopts = [ + "-lreadline", + "-rdynamic", + ], +) diff --git a/third_party/upb/bazel/ragel.BUILD b/third_party/upb/bazel/ragel.BUILD new file mode 100644 index 00000000000..5e3b24913df --- /dev/null +++ b/third_party/upb/bazel/ragel.BUILD @@ -0,0 +1,193 @@ + +package( + default_visibility = ["//visibility:public"], +) + +cc_binary( + name = "ragelc", + srcs = [ + "ragel/rubycodegen.cpp", + "ragel/goipgoto.h", + "ragel/cdtable.h", + "ragel/rubycodegen.h", + "ragel/gotable.h", + "ragel/gocodegen.cpp", + "ragel/rubyfflat.cpp", + "ragel/common.cpp", + "ragel/gofflat.cpp", + "ragel/cdtable.cpp", + "ragel/cdsplit.cpp", + "ragel/rlparse.cpp", + "ragel/csfgoto.cpp", + "ragel/javacodegen.cpp", + "ragel/gocodegen.h", + "ragel/mlgoto.cpp", + "ragel/fsmgraph.cpp", + "ragel/version.h", + "ragel/mlfflat.h", + "ragel/fsmgraph.h", + "ragel/fsmbase.cpp", + "ragel/fsmstate.cpp", + "ragel/gotablish.cpp", + "ragel/rubyflat.cpp", + "ragel/cdfgoto.h", + "ragel/cscodegen.h", + "ragel/mlflat.cpp", + "ragel/rubyflat.h", + "ragel/goftable.h", + "ragel/rbxgoto.cpp", + "ragel/csfflat.cpp", + "ragel/gofgoto.cpp", + "ragel/gofgoto.h", + "ragel/ragel.h", + "ragel/goftable.cpp", + "ragel/cdcodegen.cpp", + "ragel/rlparse.h", + "ragel/cdsplit.h", + "ragel/xmlcodegen.cpp", + "ragel/goipgoto.cpp", + "ragel/dotcodegen.h", + "ragel/gogoto.cpp", + "ragel/csflat.h", + "ragel/csfflat.h", + #"ragel/config.h.in", + "ragel/csipgoto.cpp", + "ragel/mltable.cpp", + "ragel/mlflat.h", + "ragel/csftable.cpp", + "ragel/cdgoto.h", + "ragel/goflat.cpp", + "ragel/rubyfflat.h", + "ragel/mlftable.h", + "ragel/rubyftable.h", + "ragel/fsmap.cpp", + "ragel/redfsm.cpp", + "ragel/goflat.h", + "ragel/parsetree.cpp", + "ragel/fsmmin.cpp", + "ragel/dotcodegen.cpp", + "ragel/redfsm.h", + "ragel/mlcodegen.cpp", + "ragel/cdfgoto.cpp", + "ragel/cssplit.cpp", + "ragel/cstable.cpp", + "ragel/javacodegen.h", + "ragel/parsedata.cpp", + "ragel/buffer.h", + "ragel/gogoto.h", + "ragel/csgoto.h", + "ragel/pcheck.h", + "ragel/rubyftable.cpp", + "ragel/csfgoto.h", + "ragel/common.h", + "ragel/cdftable.h", + "ragel/mlgoto.h", + "ragel/csgoto.cpp", + "ragel/cdflat.h", + "ragel/cdipgoto.h", + "ragel/cstable.h", + "ragel/gendata.h", + "ragel/cdfflat.cpp", + "ragel/gotable.cpp", + "ragel/cdcodegen.h", + "ragel/gendata.cpp", + "ragel/rubytable.h", + "ragel/csflat.cpp", + "ragel/inputdata.h", + "ragel/inputdata.cpp", + "ragel/rubytable.cpp", + "ragel/fsmattach.cpp", + "ragel/csipgoto.h", + "ragel/cscodegen.cpp", + "ragel/cdfflat.h", + "ragel/rbxgoto.h", + "ragel/xmlcodegen.h", + "ragel/gofflat.h", + "ragel/parsedata.h", + "ragel/mlfgoto.h", + "ragel/cdflat.cpp", + "ragel/config.h", + "ragel/rlscan.cpp", + "ragel/mlcodegen.h", + "ragel/mlfflat.cpp", + "ragel/mlftable.cpp", + "ragel/mltable.h", + "ragel/cdipgoto.cpp", + "ragel/cdftable.cpp", + "ragel/parsetree.h", + "ragel/rlscan.h", + "ragel/main.cpp", + "ragel/cssplit.h", + "ragel/mlfgoto.cpp", + "ragel/csftable.h", + "ragel/gotablish.h", + "ragel/cdgoto.cpp", + "aapl/avlmelkey.h", + "aapl/dlistmel.h", + "aapl/avliset.h", + "aapl/avlkeyless.h", + "aapl/sbstset.h", + "aapl/sbsttable.h", + "aapl/quicksort.h", + "aapl/avlitree.h", + "aapl/avlcommon.h", + "aapl/bstset.h", + "aapl/avlmel.h", + "aapl/insertsort.h", + "aapl/dlist.h", + "aapl/avlmap.h", + "aapl/mergesort.h", + "aapl/resize.h", + "aapl/bstcommon.h", + "aapl/bstmap.h", + "aapl/compare.h", + "aapl/svector.h", + "aapl/avlset.h", + "aapl/bsttable.h", + "aapl/avlikeyless.h", + "aapl/bubblesort.h", + "aapl/table.h", + "aapl/avlbasic.h", + "aapl/vector.h", + "aapl/avlimap.h", + "aapl/dlistval.h", + "aapl/dlcommon.h", + "aapl/avlibasic.h", + "aapl/sbstmap.h", + "aapl/avlimel.h", + "aapl/avlimelkey.h", + "aapl/avltree.h", + ], + includes = ["ragel", "aapl"], +) + +config_h_contents = """ +#define PACKAGE "ragel" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "ragel" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "ragel 6.10" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "ragel" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "6.10" + +/* Version number of package */ +#define VERSION "6.10" +""" + +genrule( + name = "gen_config_h", + outs = ["ragel/config.h"], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % config_h_contents, +) diff --git a/third_party/upb/bazel/repository_defs.bzl b/third_party/upb/bazel/repository_defs.bzl new file mode 100644 index 00000000000..7b6e78e8b83 --- /dev/null +++ b/third_party/upb/bazel/repository_defs.bzl @@ -0,0 +1,15 @@ +# A hacky way to work around the fact that native.bazel_version is only +# available from WORKSPACE macros, not BUILD macros or rules. +# +# Hopefully we can remove this if/when this is fixed: +# https://github.com/bazelbuild/bazel/issues/8305 + +def _impl(repository_ctx): + s = "bazel_version = \"" + native.bazel_version + "\"" + repository_ctx.file("bazel_version.bzl", s) + repository_ctx.file("BUILD", "") + +bazel_version_repository = repository_rule( + implementation = _impl, + local = True, +) diff --git a/third_party/upb/bazel/upb_proto_library.bzl b/third_party/upb/bazel/upb_proto_library.bzl new file mode 100644 index 00000000000..cc6bcffaede --- /dev/null +++ b/third_party/upb/bazel/upb_proto_library.bzl @@ -0,0 +1,299 @@ +"""Public rules for using upb protos: + - upb_proto_library() + - upb_proto_reflection_library() +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +# copybara:strip_for_google3_begin +load("@bazel_skylib//lib:versions.bzl", "versions") +load("@bazel_version//:bazel_version.bzl", "bazel_version") +# copybara:strip_end + +# Generic support code ######################################################### + +_is_bazel = not hasattr(native, "genmpm") + +def _get_real_short_path(file): + # For some reason, files from other archives have short paths that look like: + # ../com_google_protobuf/google/protobuf/descriptor.proto + short_path = file.short_path + if short_path.startswith("../"): + second_slash = short_path.index("/", 3) + short_path = short_path[second_slash + 1:] + return short_path + +def _get_real_root(file): + real_short_path = _get_real_short_path(file) + return file.path[:-len(real_short_path) - 1] + +def _get_real_roots(files): + roots = {} + for file in files: + real_root = _get_real_root(file) + if real_root: + roots[real_root] = True + return roots.keys() + +def _generate_output_file(ctx, src, extension): + real_short_path = _get_real_short_path(src) + real_short_path = paths.relativize(real_short_path, ctx.label.package) + output_filename = paths.replace_extension(real_short_path, extension) + ret = ctx.actions.declare_file(output_filename) + return ret + +def _filter_none(elems): + out = [] + for elem in elems: + if elem: + out.append(elem) + return out + +def _cc_library_func(ctx, name, hdrs, srcs, dep_ccinfos): + """Like cc_library(), but callable from rules. + + Args: + ctx: Rule context. + name: Unique name used to generate output files. + hdrs: Public headers that can be #included from other rules. + srcs: C/C++ source files. + dep_ccinfos: CcInfo providers of dependencies we should build/link against. + + Returns: + CcInfo provider for this compilation. + """ + + compilation_contexts = [info.compilation_context for info in dep_ccinfos] + linking_contexts = [info.linking_context for info in dep_ccinfos] + toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + + # copybara:strip_for_google3_begin + if bazel_version == "0.24.1": + # Compatibility code until gRPC is on 0.25.2 or later. + compilation_info = cc_common.compile( + ctx = ctx, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + srcs = srcs, + hdrs = hdrs, + compilation_contexts = compilation_contexts, + ) + linking_info = cc_common.link( + ctx = ctx, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + cc_compilation_outputs = compilation_info.cc_compilation_outputs, + linking_contexts = linking_contexts, + ) + return CcInfo( + compilation_context = compilation_info.compilation_context, + linking_context = linking_info.linking_context, + ) + + if not versions.is_at_least("0.25.2", bazel_version): + fail("upb requires Bazel >=0.25.2 or 0.24.1") + + # copybara:strip_end + + blaze_only_args = {} + + if not _is_bazel: + blaze_only_args["grep_includes"] = ctx.file._grep_includes + + (compilation_context, compilation_outputs) = cc_common.compile( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + name = name, + srcs = srcs, + public_hdrs = hdrs, + compilation_contexts = compilation_contexts, + **blaze_only_args + ) + (linking_context, linking_outputs) = cc_common.create_linking_context_from_compilation_outputs( + actions = ctx.actions, + name = name, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + compilation_outputs = compilation_outputs, + linking_contexts = linking_contexts, + **blaze_only_args + ) + + return CcInfo( + compilation_context = compilation_context, + linking_context = linking_context, + ) + +# upb_proto_library / upb_proto_reflection_library shared code ################# + +GeneratedSrcsInfo = provider( + fields = { + "srcs": "list of srcs", + "hdrs": "list of hdrs", + }, +) + +_WrappedCcInfo = provider(fields = ["cc_info"]) +_WrappedGeneratedSrcsInfo = provider(fields = ["srcs"]) + +def _compile_upb_protos(ctx, proto_info, proto_sources, ext): + srcs = [_generate_output_file(ctx, name, ext + ".c") for name in proto_sources] + hdrs = [_generate_output_file(ctx, name, ext + ".h") for name in proto_sources] + transitive_sets = proto_info.transitive_descriptor_sets.to_list() + ctx.actions.run( + inputs = depset( + direct = [proto_info.direct_descriptor_set], + transitive = [proto_info.transitive_descriptor_sets], + ), + tools = [ctx.executable._upbc], + outputs = srcs + hdrs, + executable = ctx.executable._protoc, + arguments = [ + "--upb_out=" + _get_real_root(srcs[0]), + "--plugin=protoc-gen-upb=" + ctx.executable._upbc.path, + "--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets]), + ] + + [_get_real_short_path(file) for file in proto_sources], + progress_message = "Generating upb protos for :" + ctx.label.name, + ) + return GeneratedSrcsInfo(srcs = srcs, hdrs = hdrs) + +def _upb_proto_rule_impl(ctx): + if len(ctx.attr.deps) != 1: + fail("only one deps dependency allowed.") + dep = ctx.attr.deps[0] + if _WrappedCcInfo not in dep or _WrappedGeneratedSrcsInfo not in dep: + fail("proto_library rule must generate _WrappedCcInfo and " + + "_WrappedGeneratedSrcsInfo (aspect should have handled this).") + cc_info = dep[_WrappedCcInfo].cc_info + srcs = dep[_WrappedGeneratedSrcsInfo].srcs + if (type(cc_info.linking_context.libraries_to_link) == "list"): + lib = cc_info.linking_context.libraries_to_link[0] + else: + lib = cc_info.linking_context.libraries_to_link.to_list()[0] + files = _filter_none([ + lib.static_library, + lib.pic_static_library, + lib.dynamic_library, + ]) + return [ + DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs)), + srcs, + cc_info, + ] + +def _upb_proto_aspect_impl(target, ctx): + proto_info = target[ProtoInfo] + files = _compile_upb_protos(ctx, proto_info, proto_info.direct_sources, ctx.attr._ext) + deps = ctx.rule.attr.deps + ctx.attr._upb + dep_ccinfos = [dep[CcInfo] for dep in deps if CcInfo in dep] + dep_ccinfos += [dep[_WrappedCcInfo].cc_info for dep in deps if _WrappedCcInfo in dep] + cc_info = _cc_library_func( + ctx = ctx, + name = ctx.rule.attr.name + ctx.attr._ext, + hdrs = files.hdrs, + srcs = files.srcs, + dep_ccinfos = dep_ccinfos, + ) + return [_WrappedCcInfo(cc_info = cc_info), _WrappedGeneratedSrcsInfo(srcs = files)] + +def _maybe_add(d): + if not _is_bazel: + d["_grep_includes"] = attr.label( + allow_single_file = True, + cfg = "host", + default = "//tools/cpp:grep-includes", + ) + return d + +# upb_proto_library() ########################################################## + +_upb_proto_library_aspect = aspect( + attrs = _maybe_add({ + "_upbc": attr.label( + executable = True, + cfg = "host", + default = "//:protoc-gen-upb", + ), + "_protoc": attr.label( + executable = True, + cfg = "host", + default = "@com_google_protobuf//:protoc", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + "_upb": attr.label_list(default = [ + "//:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + "//:upb" + ]), + "_ext": attr.string(default = ".upb"), + }), + implementation = _upb_proto_aspect_impl, + attr_aspects = ["deps"], + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) + +upb_proto_library = rule( + output_to_genfiles = True, + implementation = _upb_proto_rule_impl, + attrs = { + "deps": attr.label_list( + aspects = [_upb_proto_library_aspect], + allow_rules = ["proto_library"], + providers = [ProtoInfo], + ), + }, +) + +# upb_proto_reflection_library() ############################################### + +_upb_proto_reflection_library_aspect = aspect( + attrs = _maybe_add({ + "_upbc": attr.label( + executable = True, + cfg = "host", + default = "//:protoc-gen-upb", + ), + "_protoc": attr.label( + executable = True, + cfg = "host", + default = "@com_google_protobuf//:protoc", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + "_upb": attr.label_list( + default = [ + "//:upb", + "//:reflection", + ], + ), + "_ext": attr.string(default = ".upbdefs"), + }), + implementation = _upb_proto_aspect_impl, + attr_aspects = ["deps"], + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) + +upb_proto_reflection_library = rule( + output_to_genfiles = True, + implementation = _upb_proto_rule_impl, + attrs = { + "deps": attr.label_list( + aspects = [_upb_proto_reflection_library_aspect], + allow_rules = ["proto_library"], + providers = [ProtoInfo], + ), + }, +) diff --git a/third_party/upb/bazel/workspace_deps.bzl b/third_party/upb/bazel/workspace_deps.bzl new file mode 100644 index 00000000000..39bf524a7a1 --- /dev/null +++ b/third_party/upb/bazel/workspace_deps.bzl @@ -0,0 +1,36 @@ + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("//bazel:repository_defs.bzl", "bazel_version_repository") + +def upb_deps(): + bazel_version_repository( + name = "bazel_version", + ) + + git_repository( + name = "absl", + commit = "070f6e47b33a2909d039e620c873204f78809492", + remote = "https://github.com/abseil/abseil-cpp.git", + shallow_since = "1541627663 -0500", + ) + + git_repository( + name = "com_google_protobuf", + remote = "https://github.com/protocolbuffers/protobuf.git", + commit = "d41002663fd04325ead28439dfd5ce2822b0d6fb", + ) + + http_archive( + name = "bazel_skylib", + strip_prefix = "bazel-skylib-master", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/master.tar.gz"], + ) + + http_archive( + name = "zlib", + build_file = "@com_google_protobuf//:third_party/zlib.BUILD", + sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", + strip_prefix = "zlib-1.2.11", + urls = ["https://zlib.net/zlib-1.2.11.tar.gz"], + ) diff --git a/third_party/upb/examples/bazel/BUILD b/third_party/upb/examples/bazel/BUILD new file mode 100644 index 00000000000..94bcfe64d4f --- /dev/null +++ b/third_party/upb/examples/bazel/BUILD @@ -0,0 +1,18 @@ + +load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library") + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) + +upb_proto_library( + name = "foo_upbproto", + deps = [":foo_proto"], +) + +cc_binary( + name = "test_binary", + srcs = ["test_binary.c"], + deps = [":foo_upbproto"], +) diff --git a/third_party/upb/examples/bazel/WORKSPACE b/third_party/upb/examples/bazel/WORKSPACE new file mode 100644 index 00000000000..5dfe68ac25a --- /dev/null +++ b/third_party/upb/examples/bazel/WORKSPACE @@ -0,0 +1,14 @@ + +workspace(name = "upb_example") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "upb", + remote = "https://github.com/protocolbuffers/upb.git", + commit = "d16bf99ac4658793748cda3251226059892b3b7b", +) + +load("@upb//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() diff --git a/third_party/upb/examples/bazel/foo.proto b/third_party/upb/examples/bazel/foo.proto new file mode 100644 index 00000000000..f68b99443dd --- /dev/null +++ b/third_party/upb/examples/bazel/foo.proto @@ -0,0 +1,7 @@ + +syntax = "proto2"; + +message Foo { + optional int64 time = 1; + optional string greeting = 2; +} diff --git a/third_party/upb/examples/bazel/test_binary.c b/third_party/upb/examples/bazel/test_binary.c new file mode 100644 index 00000000000..78f367a3707 --- /dev/null +++ b/third_party/upb/examples/bazel/test_binary.c @@ -0,0 +1,17 @@ + +#include + +#include "foo.upb.h" + +int main() { + upb_arena *arena = upb_arena_new(); + Foo* foo = Foo_new(arena); + const char greeting[] = "Hello, World!\n"; + + Foo_set_time(foo, time(NULL)); + /* Warning: the proto will not copy this, the string data must outlive + * the proto. */ + Foo_set_greeting(foo, upb_strview_makez(greeting)); + + upb_arena_free(arena); +} diff --git a/third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.c b/third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.c new file mode 100644 index 00000000000..61b9299bb43 --- /dev/null +++ b/third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.c @@ -0,0 +1,485 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * google/protobuf/descriptor.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#include +#include "upb/msg.h" +#include "google/protobuf/descriptor.upb.h" + +#include "upb/port_def.inc" + +static const upb_msglayout *const google_protobuf_FileDescriptorSet_submsgs[1] = { + &google_protobuf_FileDescriptorProto_msginit, +}; + +static const upb_msglayout_field google_protobuf_FileDescriptorSet__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_FileDescriptorSet_msginit = { + &google_protobuf_FileDescriptorSet_submsgs[0], + &google_protobuf_FileDescriptorSet__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_protobuf_FileDescriptorProto_submsgs[6] = { + &google_protobuf_DescriptorProto_msginit, + &google_protobuf_EnumDescriptorProto_msginit, + &google_protobuf_FieldDescriptorProto_msginit, + &google_protobuf_FileOptions_msginit, + &google_protobuf_ServiceDescriptorProto_msginit, + &google_protobuf_SourceCodeInfo_msginit, +}; + +static const upb_msglayout_field google_protobuf_FileDescriptorProto__fields[12] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(12, 24), 2, 0, 9, 1}, + {3, UPB_SIZE(36, 72), 0, 0, 9, 3}, + {4, UPB_SIZE(40, 80), 0, 0, 11, 3}, + {5, UPB_SIZE(44, 88), 0, 1, 11, 3}, + {6, UPB_SIZE(48, 96), 0, 4, 11, 3}, + {7, UPB_SIZE(52, 104), 0, 2, 11, 3}, + {8, UPB_SIZE(28, 56), 4, 3, 11, 1}, + {9, UPB_SIZE(32, 64), 5, 5, 11, 1}, + {10, UPB_SIZE(56, 112), 0, 0, 5, 3}, + {11, UPB_SIZE(60, 120), 0, 0, 5, 3}, + {12, UPB_SIZE(20, 40), 3, 0, 9, 1}, +}; + +const upb_msglayout google_protobuf_FileDescriptorProto_msginit = { + &google_protobuf_FileDescriptorProto_submsgs[0], + &google_protobuf_FileDescriptorProto__fields[0], + UPB_SIZE(64, 128), 12, false, +}; + +static const upb_msglayout *const google_protobuf_DescriptorProto_submsgs[8] = { + &google_protobuf_DescriptorProto_msginit, + &google_protobuf_DescriptorProto_ExtensionRange_msginit, + &google_protobuf_DescriptorProto_ReservedRange_msginit, + &google_protobuf_EnumDescriptorProto_msginit, + &google_protobuf_FieldDescriptorProto_msginit, + &google_protobuf_MessageOptions_msginit, + &google_protobuf_OneofDescriptorProto_msginit, +}; + +static const upb_msglayout_field google_protobuf_DescriptorProto__fields[10] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 4, 11, 3}, + {3, UPB_SIZE(20, 40), 0, 0, 11, 3}, + {4, UPB_SIZE(24, 48), 0, 3, 11, 3}, + {5, UPB_SIZE(28, 56), 0, 1, 11, 3}, + {6, UPB_SIZE(32, 64), 0, 4, 11, 3}, + {7, UPB_SIZE(12, 24), 2, 5, 11, 1}, + {8, UPB_SIZE(36, 72), 0, 6, 11, 3}, + {9, UPB_SIZE(40, 80), 0, 2, 11, 3}, + {10, UPB_SIZE(44, 88), 0, 0, 9, 3}, +}; + +const upb_msglayout google_protobuf_DescriptorProto_msginit = { + &google_protobuf_DescriptorProto_submsgs[0], + &google_protobuf_DescriptorProto__fields[0], + UPB_SIZE(48, 96), 10, false, +}; + +static const upb_msglayout *const google_protobuf_DescriptorProto_ExtensionRange_submsgs[1] = { + &google_protobuf_ExtensionRangeOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_DescriptorProto_ExtensionRange__fields[3] = { + {1, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {2, UPB_SIZE(8, 8), 2, 0, 5, 1}, + {3, UPB_SIZE(12, 16), 3, 0, 11, 1}, +}; + +const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit = { + &google_protobuf_DescriptorProto_ExtensionRange_submsgs[0], + &google_protobuf_DescriptorProto_ExtensionRange__fields[0], + UPB_SIZE(16, 24), 3, false, +}; + +static const upb_msglayout_field google_protobuf_DescriptorProto_ReservedRange__fields[2] = { + {1, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {2, UPB_SIZE(8, 8), 2, 0, 5, 1}, +}; + +const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit = { + NULL, + &google_protobuf_DescriptorProto_ReservedRange__fields[0], + UPB_SIZE(12, 12), 2, false, +}; + +static const upb_msglayout *const google_protobuf_ExtensionRangeOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_ExtensionRangeOptions__fields[1] = { + {999, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit = { + &google_protobuf_ExtensionRangeOptions_submsgs[0], + &google_protobuf_ExtensionRangeOptions__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_protobuf_FieldDescriptorProto_submsgs[1] = { + &google_protobuf_FieldOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_FieldDescriptorProto__fields[10] = { + {1, UPB_SIZE(32, 32), 5, 0, 9, 1}, + {2, UPB_SIZE(40, 48), 6, 0, 9, 1}, + {3, UPB_SIZE(24, 24), 3, 0, 5, 1}, + {4, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {5, UPB_SIZE(16, 16), 2, 0, 14, 1}, + {6, UPB_SIZE(48, 64), 7, 0, 9, 1}, + {7, UPB_SIZE(56, 80), 8, 0, 9, 1}, + {8, UPB_SIZE(72, 112), 10, 0, 11, 1}, + {9, UPB_SIZE(28, 28), 4, 0, 5, 1}, + {10, UPB_SIZE(64, 96), 9, 0, 9, 1}, +}; + +const upb_msglayout google_protobuf_FieldDescriptorProto_msginit = { + &google_protobuf_FieldDescriptorProto_submsgs[0], + &google_protobuf_FieldDescriptorProto__fields[0], + UPB_SIZE(80, 128), 10, false, +}; + +static const upb_msglayout *const google_protobuf_OneofDescriptorProto_submsgs[1] = { + &google_protobuf_OneofOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_OneofDescriptorProto__fields[2] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(12, 24), 2, 0, 11, 1}, +}; + +const upb_msglayout google_protobuf_OneofDescriptorProto_msginit = { + &google_protobuf_OneofDescriptorProto_submsgs[0], + &google_protobuf_OneofDescriptorProto__fields[0], + UPB_SIZE(16, 32), 2, false, +}; + +static const upb_msglayout *const google_protobuf_EnumDescriptorProto_submsgs[3] = { + &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, + &google_protobuf_EnumOptions_msginit, + &google_protobuf_EnumValueDescriptorProto_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumDescriptorProto__fields[5] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 2, 11, 3}, + {3, UPB_SIZE(12, 24), 2, 1, 11, 1}, + {4, UPB_SIZE(20, 40), 0, 0, 11, 3}, + {5, UPB_SIZE(24, 48), 0, 0, 9, 3}, +}; + +const upb_msglayout google_protobuf_EnumDescriptorProto_msginit = { + &google_protobuf_EnumDescriptorProto_submsgs[0], + &google_protobuf_EnumDescriptorProto__fields[0], + UPB_SIZE(32, 64), 5, false, +}; + +static const upb_msglayout_field google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[2] = { + {1, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {2, UPB_SIZE(8, 8), 2, 0, 5, 1}, +}; + +const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit = { + NULL, + &google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[0], + UPB_SIZE(12, 12), 2, false, +}; + +static const upb_msglayout *const google_protobuf_EnumValueDescriptorProto_submsgs[1] = { + &google_protobuf_EnumValueOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumValueDescriptorProto__fields[3] = { + {1, UPB_SIZE(8, 8), 2, 0, 9, 1}, + {2, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {3, UPB_SIZE(16, 24), 3, 0, 11, 1}, +}; + +const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit = { + &google_protobuf_EnumValueDescriptorProto_submsgs[0], + &google_protobuf_EnumValueDescriptorProto__fields[0], + UPB_SIZE(24, 32), 3, false, +}; + +static const upb_msglayout *const google_protobuf_ServiceDescriptorProto_submsgs[2] = { + &google_protobuf_MethodDescriptorProto_msginit, + &google_protobuf_ServiceOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_ServiceDescriptorProto__fields[3] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 0, 11, 3}, + {3, UPB_SIZE(12, 24), 2, 1, 11, 1}, +}; + +const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit = { + &google_protobuf_ServiceDescriptorProto_submsgs[0], + &google_protobuf_ServiceDescriptorProto__fields[0], + UPB_SIZE(24, 48), 3, false, +}; + +static const upb_msglayout *const google_protobuf_MethodDescriptorProto_submsgs[1] = { + &google_protobuf_MethodOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_MethodDescriptorProto__fields[6] = { + {1, UPB_SIZE(4, 8), 3, 0, 9, 1}, + {2, UPB_SIZE(12, 24), 4, 0, 9, 1}, + {3, UPB_SIZE(20, 40), 5, 0, 9, 1}, + {4, UPB_SIZE(28, 56), 6, 0, 11, 1}, + {5, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {6, UPB_SIZE(2, 2), 2, 0, 8, 1}, +}; + +const upb_msglayout google_protobuf_MethodDescriptorProto_msginit = { + &google_protobuf_MethodDescriptorProto_submsgs[0], + &google_protobuf_MethodDescriptorProto__fields[0], + UPB_SIZE(32, 64), 6, false, +}; + +static const upb_msglayout *const google_protobuf_FileOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_FileOptions__fields[21] = { + {1, UPB_SIZE(28, 32), 11, 0, 9, 1}, + {8, UPB_SIZE(36, 48), 12, 0, 9, 1}, + {9, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {10, UPB_SIZE(16, 16), 2, 0, 8, 1}, + {11, UPB_SIZE(44, 64), 13, 0, 9, 1}, + {16, UPB_SIZE(17, 17), 3, 0, 8, 1}, + {17, UPB_SIZE(18, 18), 4, 0, 8, 1}, + {18, UPB_SIZE(19, 19), 5, 0, 8, 1}, + {20, UPB_SIZE(20, 20), 6, 0, 8, 1}, + {23, UPB_SIZE(21, 21), 7, 0, 8, 1}, + {27, UPB_SIZE(22, 22), 8, 0, 8, 1}, + {31, UPB_SIZE(23, 23), 9, 0, 8, 1}, + {36, UPB_SIZE(52, 80), 14, 0, 9, 1}, + {37, UPB_SIZE(60, 96), 15, 0, 9, 1}, + {39, UPB_SIZE(68, 112), 16, 0, 9, 1}, + {40, UPB_SIZE(76, 128), 17, 0, 9, 1}, + {41, UPB_SIZE(84, 144), 18, 0, 9, 1}, + {42, UPB_SIZE(24, 24), 10, 0, 8, 1}, + {44, UPB_SIZE(92, 160), 19, 0, 9, 1}, + {45, UPB_SIZE(100, 176), 20, 0, 9, 1}, + {999, UPB_SIZE(108, 192), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_FileOptions_msginit = { + &google_protobuf_FileOptions_submsgs[0], + &google_protobuf_FileOptions__fields[0], + UPB_SIZE(112, 208), 21, false, +}; + +static const upb_msglayout *const google_protobuf_MessageOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_MessageOptions__fields[5] = { + {1, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {2, UPB_SIZE(2, 2), 2, 0, 8, 1}, + {3, UPB_SIZE(3, 3), 3, 0, 8, 1}, + {7, UPB_SIZE(4, 4), 4, 0, 8, 1}, + {999, UPB_SIZE(8, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_MessageOptions_msginit = { + &google_protobuf_MessageOptions_submsgs[0], + &google_protobuf_MessageOptions__fields[0], + UPB_SIZE(12, 16), 5, false, +}; + +static const upb_msglayout *const google_protobuf_FieldOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_FieldOptions__fields[7] = { + {1, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {2, UPB_SIZE(24, 24), 3, 0, 8, 1}, + {3, UPB_SIZE(25, 25), 4, 0, 8, 1}, + {5, UPB_SIZE(26, 26), 5, 0, 8, 1}, + {6, UPB_SIZE(16, 16), 2, 0, 14, 1}, + {10, UPB_SIZE(27, 27), 6, 0, 8, 1}, + {999, UPB_SIZE(28, 32), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_FieldOptions_msginit = { + &google_protobuf_FieldOptions_submsgs[0], + &google_protobuf_FieldOptions__fields[0], + UPB_SIZE(32, 40), 7, false, +}; + +static const upb_msglayout *const google_protobuf_OneofOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_OneofOptions__fields[1] = { + {999, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_OneofOptions_msginit = { + &google_protobuf_OneofOptions_submsgs[0], + &google_protobuf_OneofOptions__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_protobuf_EnumOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumOptions__fields[3] = { + {2, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {3, UPB_SIZE(2, 2), 2, 0, 8, 1}, + {999, UPB_SIZE(4, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_EnumOptions_msginit = { + &google_protobuf_EnumOptions_submsgs[0], + &google_protobuf_EnumOptions__fields[0], + UPB_SIZE(8, 16), 3, false, +}; + +static const upb_msglayout *const google_protobuf_EnumValueOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumValueOptions__fields[2] = { + {1, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {999, UPB_SIZE(4, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_EnumValueOptions_msginit = { + &google_protobuf_EnumValueOptions_submsgs[0], + &google_protobuf_EnumValueOptions__fields[0], + UPB_SIZE(8, 16), 2, false, +}; + +static const upb_msglayout *const google_protobuf_ServiceOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_ServiceOptions__fields[2] = { + {33, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {999, UPB_SIZE(4, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_ServiceOptions_msginit = { + &google_protobuf_ServiceOptions_submsgs[0], + &google_protobuf_ServiceOptions__fields[0], + UPB_SIZE(8, 16), 2, false, +}; + +static const upb_msglayout *const google_protobuf_MethodOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_MethodOptions__fields[3] = { + {33, UPB_SIZE(16, 16), 2, 0, 8, 1}, + {34, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {999, UPB_SIZE(20, 24), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_MethodOptions_msginit = { + &google_protobuf_MethodOptions_submsgs[0], + &google_protobuf_MethodOptions__fields[0], + UPB_SIZE(24, 32), 3, false, +}; + +static const upb_msglayout *const google_protobuf_UninterpretedOption_submsgs[1] = { + &google_protobuf_UninterpretedOption_NamePart_msginit, +}; + +static const upb_msglayout_field google_protobuf_UninterpretedOption__fields[7] = { + {2, UPB_SIZE(56, 80), 0, 0, 11, 3}, + {3, UPB_SIZE(32, 32), 4, 0, 9, 1}, + {4, UPB_SIZE(8, 8), 1, 0, 4, 1}, + {5, UPB_SIZE(16, 16), 2, 0, 3, 1}, + {6, UPB_SIZE(24, 24), 3, 0, 1, 1}, + {7, UPB_SIZE(40, 48), 5, 0, 12, 1}, + {8, UPB_SIZE(48, 64), 6, 0, 9, 1}, +}; + +const upb_msglayout google_protobuf_UninterpretedOption_msginit = { + &google_protobuf_UninterpretedOption_submsgs[0], + &google_protobuf_UninterpretedOption__fields[0], + UPB_SIZE(64, 96), 7, false, +}; + +static const upb_msglayout_field google_protobuf_UninterpretedOption_NamePart__fields[2] = { + {1, UPB_SIZE(4, 8), 2, 0, 9, 2}, + {2, UPB_SIZE(1, 1), 1, 0, 8, 2}, +}; + +const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit = { + NULL, + &google_protobuf_UninterpretedOption_NamePart__fields[0], + UPB_SIZE(16, 32), 2, false, +}; + +static const upb_msglayout *const google_protobuf_SourceCodeInfo_submsgs[1] = { + &google_protobuf_SourceCodeInfo_Location_msginit, +}; + +static const upb_msglayout_field google_protobuf_SourceCodeInfo__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_SourceCodeInfo_msginit = { + &google_protobuf_SourceCodeInfo_submsgs[0], + &google_protobuf_SourceCodeInfo__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout_field google_protobuf_SourceCodeInfo_Location__fields[5] = { + {1, UPB_SIZE(20, 40), 0, 0, 5, 3}, + {2, UPB_SIZE(24, 48), 0, 0, 5, 3}, + {3, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {4, UPB_SIZE(12, 24), 2, 0, 9, 1}, + {6, UPB_SIZE(28, 56), 0, 0, 9, 3}, +}; + +const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit = { + NULL, + &google_protobuf_SourceCodeInfo_Location__fields[0], + UPB_SIZE(32, 64), 5, false, +}; + +static const upb_msglayout *const google_protobuf_GeneratedCodeInfo_submsgs[1] = { + &google_protobuf_GeneratedCodeInfo_Annotation_msginit, +}; + +static const upb_msglayout_field google_protobuf_GeneratedCodeInfo__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit = { + &google_protobuf_GeneratedCodeInfo_submsgs[0], + &google_protobuf_GeneratedCodeInfo__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout_field google_protobuf_GeneratedCodeInfo_Annotation__fields[4] = { + {1, UPB_SIZE(20, 32), 0, 0, 5, 3}, + {2, UPB_SIZE(12, 16), 3, 0, 9, 1}, + {3, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {4, UPB_SIZE(8, 8), 2, 0, 5, 1}, +}; + +const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit = { + NULL, + &google_protobuf_GeneratedCodeInfo_Annotation__fields[0], + UPB_SIZE(24, 48), 4, false, +}; + +#include "upb/port_undef.inc" + diff --git a/third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.h b/third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.h new file mode 100644 index 00000000000..681614910e0 --- /dev/null +++ b/third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.h @@ -0,0 +1,1690 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * google/protobuf/descriptor.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ +#define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ + +#include "upb/generated_util.h" +#include "upb/msg.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +struct google_protobuf_FileDescriptorSet; +struct google_protobuf_FileDescriptorProto; +struct google_protobuf_DescriptorProto; +struct google_protobuf_DescriptorProto_ExtensionRange; +struct google_protobuf_DescriptorProto_ReservedRange; +struct google_protobuf_ExtensionRangeOptions; +struct google_protobuf_FieldDescriptorProto; +struct google_protobuf_OneofDescriptorProto; +struct google_protobuf_EnumDescriptorProto; +struct google_protobuf_EnumDescriptorProto_EnumReservedRange; +struct google_protobuf_EnumValueDescriptorProto; +struct google_protobuf_ServiceDescriptorProto; +struct google_protobuf_MethodDescriptorProto; +struct google_protobuf_FileOptions; +struct google_protobuf_MessageOptions; +struct google_protobuf_FieldOptions; +struct google_protobuf_OneofOptions; +struct google_protobuf_EnumOptions; +struct google_protobuf_EnumValueOptions; +struct google_protobuf_ServiceOptions; +struct google_protobuf_MethodOptions; +struct google_protobuf_UninterpretedOption; +struct google_protobuf_UninterpretedOption_NamePart; +struct google_protobuf_SourceCodeInfo; +struct google_protobuf_SourceCodeInfo_Location; +struct google_protobuf_GeneratedCodeInfo; +struct google_protobuf_GeneratedCodeInfo_Annotation; +typedef struct google_protobuf_FileDescriptorSet google_protobuf_FileDescriptorSet; +typedef struct google_protobuf_FileDescriptorProto google_protobuf_FileDescriptorProto; +typedef struct google_protobuf_DescriptorProto google_protobuf_DescriptorProto; +typedef struct google_protobuf_DescriptorProto_ExtensionRange google_protobuf_DescriptorProto_ExtensionRange; +typedef struct google_protobuf_DescriptorProto_ReservedRange google_protobuf_DescriptorProto_ReservedRange; +typedef struct google_protobuf_ExtensionRangeOptions google_protobuf_ExtensionRangeOptions; +typedef struct google_protobuf_FieldDescriptorProto google_protobuf_FieldDescriptorProto; +typedef struct google_protobuf_OneofDescriptorProto google_protobuf_OneofDescriptorProto; +typedef struct google_protobuf_EnumDescriptorProto google_protobuf_EnumDescriptorProto; +typedef struct google_protobuf_EnumDescriptorProto_EnumReservedRange google_protobuf_EnumDescriptorProto_EnumReservedRange; +typedef struct google_protobuf_EnumValueDescriptorProto google_protobuf_EnumValueDescriptorProto; +typedef struct google_protobuf_ServiceDescriptorProto google_protobuf_ServiceDescriptorProto; +typedef struct google_protobuf_MethodDescriptorProto google_protobuf_MethodDescriptorProto; +typedef struct google_protobuf_FileOptions google_protobuf_FileOptions; +typedef struct google_protobuf_MessageOptions google_protobuf_MessageOptions; +typedef struct google_protobuf_FieldOptions google_protobuf_FieldOptions; +typedef struct google_protobuf_OneofOptions google_protobuf_OneofOptions; +typedef struct google_protobuf_EnumOptions google_protobuf_EnumOptions; +typedef struct google_protobuf_EnumValueOptions google_protobuf_EnumValueOptions; +typedef struct google_protobuf_ServiceOptions google_protobuf_ServiceOptions; +typedef struct google_protobuf_MethodOptions google_protobuf_MethodOptions; +typedef struct google_protobuf_UninterpretedOption google_protobuf_UninterpretedOption; +typedef struct google_protobuf_UninterpretedOption_NamePart google_protobuf_UninterpretedOption_NamePart; +typedef struct google_protobuf_SourceCodeInfo google_protobuf_SourceCodeInfo; +typedef struct google_protobuf_SourceCodeInfo_Location google_protobuf_SourceCodeInfo_Location; +typedef struct google_protobuf_GeneratedCodeInfo google_protobuf_GeneratedCodeInfo; +typedef struct google_protobuf_GeneratedCodeInfo_Annotation google_protobuf_GeneratedCodeInfo_Annotation; +extern const upb_msglayout google_protobuf_FileDescriptorSet_msginit; +extern const upb_msglayout google_protobuf_FileDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_DescriptorProto_msginit; +extern const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit; +extern const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit; +extern const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit; +extern const upb_msglayout google_protobuf_FieldDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_OneofDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_EnumDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit; +extern const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_MethodDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_FileOptions_msginit; +extern const upb_msglayout google_protobuf_MessageOptions_msginit; +extern const upb_msglayout google_protobuf_FieldOptions_msginit; +extern const upb_msglayout google_protobuf_OneofOptions_msginit; +extern const upb_msglayout google_protobuf_EnumOptions_msginit; +extern const upb_msglayout google_protobuf_EnumValueOptions_msginit; +extern const upb_msglayout google_protobuf_ServiceOptions_msginit; +extern const upb_msglayout google_protobuf_MethodOptions_msginit; +extern const upb_msglayout google_protobuf_UninterpretedOption_msginit; +extern const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit; +extern const upb_msglayout google_protobuf_SourceCodeInfo_msginit; +extern const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit; +extern const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit; +extern const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit; + +typedef enum { + google_protobuf_FieldDescriptorProto_LABEL_OPTIONAL = 1, + google_protobuf_FieldDescriptorProto_LABEL_REQUIRED = 2, + google_protobuf_FieldDescriptorProto_LABEL_REPEATED = 3 +} google_protobuf_FieldDescriptorProto_Label; + +typedef enum { + google_protobuf_FieldDescriptorProto_TYPE_DOUBLE = 1, + google_protobuf_FieldDescriptorProto_TYPE_FLOAT = 2, + google_protobuf_FieldDescriptorProto_TYPE_INT64 = 3, + google_protobuf_FieldDescriptorProto_TYPE_UINT64 = 4, + google_protobuf_FieldDescriptorProto_TYPE_INT32 = 5, + google_protobuf_FieldDescriptorProto_TYPE_FIXED64 = 6, + google_protobuf_FieldDescriptorProto_TYPE_FIXED32 = 7, + google_protobuf_FieldDescriptorProto_TYPE_BOOL = 8, + google_protobuf_FieldDescriptorProto_TYPE_STRING = 9, + google_protobuf_FieldDescriptorProto_TYPE_GROUP = 10, + google_protobuf_FieldDescriptorProto_TYPE_MESSAGE = 11, + google_protobuf_FieldDescriptorProto_TYPE_BYTES = 12, + google_protobuf_FieldDescriptorProto_TYPE_UINT32 = 13, + google_protobuf_FieldDescriptorProto_TYPE_ENUM = 14, + google_protobuf_FieldDescriptorProto_TYPE_SFIXED32 = 15, + google_protobuf_FieldDescriptorProto_TYPE_SFIXED64 = 16, + google_protobuf_FieldDescriptorProto_TYPE_SINT32 = 17, + google_protobuf_FieldDescriptorProto_TYPE_SINT64 = 18 +} google_protobuf_FieldDescriptorProto_Type; + +typedef enum { + google_protobuf_FieldOptions_STRING = 0, + google_protobuf_FieldOptions_CORD = 1, + google_protobuf_FieldOptions_STRING_PIECE = 2 +} google_protobuf_FieldOptions_CType; + +typedef enum { + google_protobuf_FieldOptions_JS_NORMAL = 0, + google_protobuf_FieldOptions_JS_STRING = 1, + google_protobuf_FieldOptions_JS_NUMBER = 2 +} google_protobuf_FieldOptions_JSType; + +typedef enum { + google_protobuf_FileOptions_SPEED = 1, + google_protobuf_FileOptions_CODE_SIZE = 2, + google_protobuf_FileOptions_LITE_RUNTIME = 3 +} google_protobuf_FileOptions_OptimizeMode; + +typedef enum { + google_protobuf_MethodOptions_IDEMPOTENCY_UNKNOWN = 0, + google_protobuf_MethodOptions_NO_SIDE_EFFECTS = 1, + google_protobuf_MethodOptions_IDEMPOTENT = 2 +} google_protobuf_MethodOptions_IdempotencyLevel; + + +/* google.protobuf.FileDescriptorSet */ + +UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_new(upb_arena *arena) { + return (google_protobuf_FileDescriptorSet *)upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena); +} +UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FileDescriptorSet *ret = google_protobuf_FileDescriptorSet_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FileDescriptorSet_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FileDescriptorSet_serialize(const google_protobuf_FileDescriptorSet *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FileDescriptorSet_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_FileDescriptorProto* const* google_protobuf_FileDescriptorSet_file(const google_protobuf_FileDescriptorSet *msg, size_t *len) { return (const google_protobuf_FileDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_FileDescriptorProto** google_protobuf_FileDescriptorSet_mutable_file(google_protobuf_FileDescriptorSet *msg, size_t *len) { + return (google_protobuf_FileDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_FileDescriptorProto** google_protobuf_FileDescriptorSet_resize_file(google_protobuf_FileDescriptorSet *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FileDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FileDescriptorProto* google_protobuf_FileDescriptorSet_add_file(google_protobuf_FileDescriptorSet *msg, upb_arena *arena) { + struct google_protobuf_FileDescriptorProto* sub = (struct google_protobuf_FileDescriptorProto*)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.FileDescriptorProto */ + +UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_FileDescriptorProto *)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FileDescriptorProto *ret = google_protobuf_FileDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FileDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FileDescriptorProto_serialize(const google_protobuf_FileDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FileDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_name(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_FileDescriptorProto_name(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_package(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_FileDescriptorProto_package(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)); } +UPB_INLINE upb_strview const* google_protobuf_FileDescriptorProto_dependency(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(36, 72), len); } +UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_FileDescriptorProto_message_type(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(40, 80), len); } +UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_FileDescriptorProto_enum_type(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(44, 88), len); } +UPB_INLINE const google_protobuf_ServiceDescriptorProto* const* google_protobuf_FileDescriptorProto_service(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_ServiceDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(48, 96), len); } +UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_FileDescriptorProto_extension(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_FieldDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(52, 104), len); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_options(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE const google_protobuf_FileOptions* google_protobuf_FileDescriptorProto_options(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_FileOptions*, UPB_SIZE(28, 56)); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_source_code_info(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE const google_protobuf_SourceCodeInfo* google_protobuf_FileDescriptorProto_source_code_info(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_SourceCodeInfo*, UPB_SIZE(32, 64)); } +UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_public_dependency(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(56, 112), len); } +UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_weak_dependency(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(60, 120), len); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_syntax(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE upb_strview google_protobuf_FileDescriptorProto_syntax(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)); } + +UPB_INLINE void google_protobuf_FileDescriptorProto_set_name(google_protobuf_FileDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_package(google_protobuf_FileDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE upb_strview* google_protobuf_FileDescriptorProto_mutable_dependency(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(36, 72), len); +} +UPB_INLINE upb_strview* google_protobuf_FileDescriptorProto_resize_dependency(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(36, 72), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_FileDescriptorProto_add_dependency(google_protobuf_FileDescriptorProto *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(36, 72), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_FileDescriptorProto_mutable_message_type(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(40, 80), len); +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_FileDescriptorProto_resize_message_type(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(40, 80), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto* google_protobuf_FileDescriptorProto_add_message_type(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto* sub = (struct google_protobuf_DescriptorProto*)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(40, 80), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_FileDescriptorProto_mutable_enum_type(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(44, 88), len); +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_FileDescriptorProto_resize_enum_type(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(44, 88), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumDescriptorProto* google_protobuf_FileDescriptorProto_add_enum_type(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumDescriptorProto* sub = (struct google_protobuf_EnumDescriptorProto*)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(44, 88), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_ServiceDescriptorProto** google_protobuf_FileDescriptorProto_mutable_service(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_ServiceDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(48, 96), len); +} +UPB_INLINE google_protobuf_ServiceDescriptorProto** google_protobuf_FileDescriptorProto_resize_service(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_ServiceDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(48, 96), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_ServiceDescriptorProto* google_protobuf_FileDescriptorProto_add_service(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_ServiceDescriptorProto* sub = (struct google_protobuf_ServiceDescriptorProto*)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(48, 96), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_FileDescriptorProto_mutable_extension(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(52, 104), len); +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_FileDescriptorProto_resize_extension(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(52, 104), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FieldDescriptorProto* google_protobuf_FileDescriptorProto_add_extension(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldDescriptorProto* sub = (struct google_protobuf_FieldDescriptorProto*)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(52, 104), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_options(google_protobuf_FileDescriptorProto *msg, google_protobuf_FileOptions* value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, google_protobuf_FileOptions*, UPB_SIZE(28, 56)) = value; +} +UPB_INLINE struct google_protobuf_FileOptions* google_protobuf_FileDescriptorProto_mutable_options(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FileOptions* sub = (struct google_protobuf_FileOptions*)google_protobuf_FileDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_FileOptions*)upb_msg_new(&google_protobuf_FileOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_FileDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_source_code_info(google_protobuf_FileDescriptorProto *msg, google_protobuf_SourceCodeInfo* value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, google_protobuf_SourceCodeInfo*, UPB_SIZE(32, 64)) = value; +} +UPB_INLINE struct google_protobuf_SourceCodeInfo* google_protobuf_FileDescriptorProto_mutable_source_code_info(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_SourceCodeInfo* sub = (struct google_protobuf_SourceCodeInfo*)google_protobuf_FileDescriptorProto_source_code_info(msg); + if (sub == NULL) { + sub = (struct google_protobuf_SourceCodeInfo*)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); + if (!sub) return NULL; + google_protobuf_FileDescriptorProto_set_source_code_info(msg, sub); + } + return sub; +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_mutable_public_dependency(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(56, 112), len); +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_resize_public_dependency(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(56, 112), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_FileDescriptorProto_add_public_dependency(google_protobuf_FileDescriptorProto *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(56, 112), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_mutable_weak_dependency(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(60, 120), len); +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_resize_weak_dependency(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(60, 120), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_FileDescriptorProto_add_weak_dependency(google_protobuf_FileDescriptorProto *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(60, 120), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_syntax(google_protobuf_FileDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)) = value; +} + +/* google.protobuf.DescriptorProto */ + +UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_new(upb_arena *arena) { + return (google_protobuf_DescriptorProto *)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_DescriptorProto *ret = google_protobuf_DescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_DescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_DescriptorProto_serialize(const google_protobuf_DescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_DescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_DescriptorProto_has_name(const google_protobuf_DescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_DescriptorProto_name(const google_protobuf_DescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_DescriptorProto_field(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_FieldDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_DescriptorProto_nested_type(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } +UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_DescriptorProto_enum_type(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(24, 48), len); } +UPB_INLINE const google_protobuf_DescriptorProto_ExtensionRange* const* google_protobuf_DescriptorProto_extension_range(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto_ExtensionRange* const*)_upb_array_accessor(msg, UPB_SIZE(28, 56), len); } +UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_DescriptorProto_extension(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_FieldDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(32, 64), len); } +UPB_INLINE bool google_protobuf_DescriptorProto_has_options(const google_protobuf_DescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_MessageOptions* google_protobuf_DescriptorProto_options(const google_protobuf_DescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_MessageOptions*, UPB_SIZE(12, 24)); } +UPB_INLINE const google_protobuf_OneofDescriptorProto* const* google_protobuf_DescriptorProto_oneof_decl(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_OneofDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(36, 72), len); } +UPB_INLINE const google_protobuf_DescriptorProto_ReservedRange* const* google_protobuf_DescriptorProto_reserved_range(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto_ReservedRange* const*)_upb_array_accessor(msg, UPB_SIZE(40, 80), len); } +UPB_INLINE upb_strview const* google_protobuf_DescriptorProto_reserved_name(const google_protobuf_DescriptorProto *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(44, 88), len); } + +UPB_INLINE void google_protobuf_DescriptorProto_set_name(google_protobuf_DescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_mutable_field(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_resize_field(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FieldDescriptorProto* google_protobuf_DescriptorProto_add_field(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldDescriptorProto* sub = (struct google_protobuf_FieldDescriptorProto*)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_DescriptorProto_mutable_nested_type(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_DescriptorProto_resize_nested_type(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto* google_protobuf_DescriptorProto_add_nested_type(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto* sub = (struct google_protobuf_DescriptorProto*)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_DescriptorProto_mutable_enum_type(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(24, 48), len); +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_DescriptorProto_resize_enum_type(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(24, 48), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumDescriptorProto* google_protobuf_DescriptorProto_add_enum_type(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumDescriptorProto* sub = (struct google_protobuf_EnumDescriptorProto*)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(24, 48), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange** google_protobuf_DescriptorProto_mutable_extension_range(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto_ExtensionRange**)_upb_array_mutable_accessor(msg, UPB_SIZE(28, 56), len); +} +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange** google_protobuf_DescriptorProto_resize_extension_range(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto_ExtensionRange**)_upb_array_resize_accessor(msg, UPB_SIZE(28, 56), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto_ExtensionRange* google_protobuf_DescriptorProto_add_extension_range(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto_ExtensionRange* sub = (struct google_protobuf_DescriptorProto_ExtensionRange*)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(28, 56), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_mutable_extension(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(32, 64), len); +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_resize_extension(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(32, 64), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FieldDescriptorProto* google_protobuf_DescriptorProto_add_extension(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldDescriptorProto* sub = (struct google_protobuf_FieldDescriptorProto*)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(32, 64), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_DescriptorProto_set_options(google_protobuf_DescriptorProto *msg, google_protobuf_MessageOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_MessageOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_MessageOptions* google_protobuf_DescriptorProto_mutable_options(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_MessageOptions* sub = (struct google_protobuf_MessageOptions*)google_protobuf_DescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_MessageOptions*)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_DescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE google_protobuf_OneofDescriptorProto** google_protobuf_DescriptorProto_mutable_oneof_decl(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_OneofDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(36, 72), len); +} +UPB_INLINE google_protobuf_OneofDescriptorProto** google_protobuf_DescriptorProto_resize_oneof_decl(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_OneofDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(36, 72), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_OneofDescriptorProto* google_protobuf_DescriptorProto_add_oneof_decl(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_OneofDescriptorProto* sub = (struct google_protobuf_OneofDescriptorProto*)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(36, 72), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange** google_protobuf_DescriptorProto_mutable_reserved_range(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto_ReservedRange**)_upb_array_mutable_accessor(msg, UPB_SIZE(40, 80), len); +} +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange** google_protobuf_DescriptorProto_resize_reserved_range(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto_ReservedRange**)_upb_array_resize_accessor(msg, UPB_SIZE(40, 80), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto_ReservedRange* google_protobuf_DescriptorProto_add_reserved_range(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto_ReservedRange* sub = (struct google_protobuf_DescriptorProto_ReservedRange*)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(40, 80), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE upb_strview* google_protobuf_DescriptorProto_mutable_reserved_name(google_protobuf_DescriptorProto *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(44, 88), len); +} +UPB_INLINE upb_strview* google_protobuf_DescriptorProto_resize_reserved_name(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(44, 88), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_DescriptorProto_add_reserved_name(google_protobuf_DescriptorProto *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(44, 88), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} + +/* google.protobuf.DescriptorProto.ExtensionRange */ + +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_new(upb_arena *arena) { + return (google_protobuf_DescriptorProto_ExtensionRange *)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); +} +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_DescriptorProto_ExtensionRange *ret = google_protobuf_DescriptorProto_ExtensionRange_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_DescriptorProto_ExtensionRange_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_DescriptorProto_ExtensionRange_serialize(const google_protobuf_DescriptorProto_ExtensionRange *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_DescriptorProto_ExtensionRange_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_DescriptorProto_ExtensionRange_has_start(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ExtensionRange_start(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_DescriptorProto_ExtensionRange_has_end(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ExtensionRange_end(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_DescriptorProto_ExtensionRange_has_options(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE const google_protobuf_ExtensionRangeOptions* google_protobuf_DescriptorProto_ExtensionRange_options(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return UPB_FIELD_AT(msg, const google_protobuf_ExtensionRangeOptions*, UPB_SIZE(12, 16)); } + +UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_start(google_protobuf_DescriptorProto_ExtensionRange *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_end(google_protobuf_DescriptorProto_ExtensionRange *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_options(google_protobuf_DescriptorProto_ExtensionRange *msg, google_protobuf_ExtensionRangeOptions* value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, google_protobuf_ExtensionRangeOptions*, UPB_SIZE(12, 16)) = value; +} +UPB_INLINE struct google_protobuf_ExtensionRangeOptions* google_protobuf_DescriptorProto_ExtensionRange_mutable_options(google_protobuf_DescriptorProto_ExtensionRange *msg, upb_arena *arena) { + struct google_protobuf_ExtensionRangeOptions* sub = (struct google_protobuf_ExtensionRangeOptions*)google_protobuf_DescriptorProto_ExtensionRange_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_ExtensionRangeOptions*)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_DescriptorProto_ExtensionRange_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.DescriptorProto.ReservedRange */ + +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_new(upb_arena *arena) { + return (google_protobuf_DescriptorProto_ReservedRange *)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); +} +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_DescriptorProto_ReservedRange *ret = google_protobuf_DescriptorProto_ReservedRange_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_DescriptorProto_ReservedRange_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_DescriptorProto_ReservedRange_serialize(const google_protobuf_DescriptorProto_ReservedRange *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_DescriptorProto_ReservedRange_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_DescriptorProto_ReservedRange_has_start(const google_protobuf_DescriptorProto_ReservedRange *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ReservedRange_start(const google_protobuf_DescriptorProto_ReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_DescriptorProto_ReservedRange_has_end(const google_protobuf_DescriptorProto_ReservedRange *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ReservedRange_end(const google_protobuf_DescriptorProto_ReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_start(google_protobuf_DescriptorProto_ReservedRange *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_end(google_protobuf_DescriptorProto_ReservedRange *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +/* google.protobuf.ExtensionRangeOptions */ + +UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_new(upb_arena *arena) { + return (google_protobuf_ExtensionRangeOptions *)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); +} +UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_ExtensionRangeOptions *ret = google_protobuf_ExtensionRangeOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_ExtensionRangeOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_ExtensionRangeOptions_serialize(const google_protobuf_ExtensionRangeOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_ExtensionRangeOptions_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_ExtensionRangeOptions_uninterpreted_option(const google_protobuf_ExtensionRangeOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ExtensionRangeOptions_mutable_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ExtensionRangeOptions_resize_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_ExtensionRangeOptions_add_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.FieldDescriptorProto */ + +UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto *)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FieldDescriptorProto *ret = google_protobuf_FieldDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FieldDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FieldDescriptorProto_serialize(const google_protobuf_FieldDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FieldDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_name(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_name(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_extendee(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_extendee(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_number(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_number(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_label(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_label(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_type(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_type(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_type_name(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 7); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_type_name(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_default_value(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 8); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_default_value(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(56, 80)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_options(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 10); } +UPB_INLINE const google_protobuf_FieldOptions* google_protobuf_FieldDescriptorProto_options(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_FieldOptions*, UPB_SIZE(72, 112)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_oneof_index(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_oneof_index(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(28, 28)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_json_name(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 9); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_json_name(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(64, 96)); } + +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_name(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_extendee(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_number(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_label(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_type(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_type_name(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 7); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_default_value(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 8); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(56, 80)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_options(google_protobuf_FieldDescriptorProto *msg, google_protobuf_FieldOptions* value) { + _upb_sethas(msg, 10); + UPB_FIELD_AT(msg, google_protobuf_FieldOptions*, UPB_SIZE(72, 112)) = value; +} +UPB_INLINE struct google_protobuf_FieldOptions* google_protobuf_FieldDescriptorProto_mutable_options(google_protobuf_FieldDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldOptions* sub = (struct google_protobuf_FieldOptions*)google_protobuf_FieldDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_FieldOptions*)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_FieldDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_oneof_index(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(28, 28)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_json_name(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 9); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(64, 96)) = value; +} + +/* google.protobuf.OneofDescriptorProto */ + +UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_OneofDescriptorProto *)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_OneofDescriptorProto *ret = google_protobuf_OneofDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_OneofDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_OneofDescriptorProto_serialize(const google_protobuf_OneofDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_OneofDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_OneofDescriptorProto_has_name(const google_protobuf_OneofDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_OneofDescriptorProto_name(const google_protobuf_OneofDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_OneofDescriptorProto_has_options(const google_protobuf_OneofDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_OneofOptions* google_protobuf_OneofDescriptorProto_options(const google_protobuf_OneofDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_OneofOptions*, UPB_SIZE(12, 24)); } + +UPB_INLINE void google_protobuf_OneofDescriptorProto_set_name(google_protobuf_OneofDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_OneofDescriptorProto_set_options(google_protobuf_OneofDescriptorProto *msg, google_protobuf_OneofOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_OneofOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_OneofOptions* google_protobuf_OneofDescriptorProto_mutable_options(google_protobuf_OneofDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_OneofOptions* sub = (struct google_protobuf_OneofOptions*)google_protobuf_OneofDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_OneofOptions*)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_OneofDescriptorProto_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.EnumDescriptorProto */ + +UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto *)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumDescriptorProto *ret = google_protobuf_EnumDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumDescriptorProto_serialize(const google_protobuf_EnumDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_name(const google_protobuf_EnumDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_EnumDescriptorProto_name(const google_protobuf_EnumDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE const google_protobuf_EnumValueDescriptorProto* const* google_protobuf_EnumDescriptorProto_value(const google_protobuf_EnumDescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumValueDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_options(const google_protobuf_EnumDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_EnumOptions* google_protobuf_EnumDescriptorProto_options(const google_protobuf_EnumDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_EnumOptions*, UPB_SIZE(12, 24)); } +UPB_INLINE const google_protobuf_EnumDescriptorProto_EnumReservedRange* const* google_protobuf_EnumDescriptorProto_reserved_range(const google_protobuf_EnumDescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumDescriptorProto_EnumReservedRange* const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } +UPB_INLINE upb_strview const* google_protobuf_EnumDescriptorProto_reserved_name(const google_protobuf_EnumDescriptorProto *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(24, 48), len); } + +UPB_INLINE void google_protobuf_EnumDescriptorProto_set_name(google_protobuf_EnumDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE google_protobuf_EnumValueDescriptorProto** google_protobuf_EnumDescriptorProto_mutable_value(google_protobuf_EnumDescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumValueDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE google_protobuf_EnumValueDescriptorProto** google_protobuf_EnumDescriptorProto_resize_value(google_protobuf_EnumDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumValueDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumValueDescriptorProto* google_protobuf_EnumDescriptorProto_add_value(google_protobuf_EnumDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumValueDescriptorProto* sub = (struct google_protobuf_EnumValueDescriptorProto*)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_EnumDescriptorProto_set_options(google_protobuf_EnumDescriptorProto *msg, google_protobuf_EnumOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_EnumOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_EnumOptions* google_protobuf_EnumDescriptorProto_mutable_options(google_protobuf_EnumDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumOptions* sub = (struct google_protobuf_EnumOptions*)google_protobuf_EnumDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_EnumOptions*)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_EnumDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange** google_protobuf_EnumDescriptorProto_mutable_reserved_range(google_protobuf_EnumDescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumDescriptorProto_EnumReservedRange**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange** google_protobuf_EnumDescriptorProto_resize_reserved_range(google_protobuf_EnumDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto_EnumReservedRange**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumDescriptorProto_EnumReservedRange* google_protobuf_EnumDescriptorProto_add_reserved_range(google_protobuf_EnumDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumDescriptorProto_EnumReservedRange* sub = (struct google_protobuf_EnumDescriptorProto_EnumReservedRange*)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE upb_strview* google_protobuf_EnumDescriptorProto_mutable_reserved_name(google_protobuf_EnumDescriptorProto *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(24, 48), len); +} +UPB_INLINE upb_strview* google_protobuf_EnumDescriptorProto_resize_reserved_name(google_protobuf_EnumDescriptorProto *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(24, 48), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_EnumDescriptorProto_add_reserved_name(google_protobuf_EnumDescriptorProto *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(24, 48), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} + +/* google.protobuf.EnumDescriptorProto.EnumReservedRange */ + +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_new(upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto_EnumReservedRange *)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); +} +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumDescriptorProto_EnumReservedRange *ret = google_protobuf_EnumDescriptorProto_EnumReservedRange_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumDescriptorProto_EnumReservedRange_serialize(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumDescriptorProto_EnumReservedRange_has_start(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_EnumDescriptorProto_EnumReservedRange_start(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_EnumDescriptorProto_EnumReservedRange_has_end(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_EnumDescriptorProto_EnumReservedRange_end(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_start(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_end(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +/* google.protobuf.EnumValueDescriptorProto */ + +UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_EnumValueDescriptorProto *)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumValueDescriptorProto *ret = google_protobuf_EnumValueDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumValueDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumValueDescriptorProto_serialize(const google_protobuf_EnumValueDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumValueDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumValueDescriptorProto_has_name(const google_protobuf_EnumValueDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_EnumValueDescriptorProto_name(const google_protobuf_EnumValueDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_EnumValueDescriptorProto_has_number(const google_protobuf_EnumValueDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_EnumValueDescriptorProto_number(const google_protobuf_EnumValueDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_EnumValueDescriptorProto_has_options(const google_protobuf_EnumValueDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE const google_protobuf_EnumValueOptions* google_protobuf_EnumValueDescriptorProto_options(const google_protobuf_EnumValueDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_EnumValueOptions*, UPB_SIZE(16, 24)); } + +UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_name(google_protobuf_EnumValueDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_number(google_protobuf_EnumValueDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_options(google_protobuf_EnumValueDescriptorProto *msg, google_protobuf_EnumValueOptions* value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, google_protobuf_EnumValueOptions*, UPB_SIZE(16, 24)) = value; +} +UPB_INLINE struct google_protobuf_EnumValueOptions* google_protobuf_EnumValueDescriptorProto_mutable_options(google_protobuf_EnumValueDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumValueOptions* sub = (struct google_protobuf_EnumValueOptions*)google_protobuf_EnumValueDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_EnumValueOptions*)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_EnumValueDescriptorProto_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.ServiceDescriptorProto */ + +UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_ServiceDescriptorProto *)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_ServiceDescriptorProto *ret = google_protobuf_ServiceDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_ServiceDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_ServiceDescriptorProto_serialize(const google_protobuf_ServiceDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_ServiceDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_ServiceDescriptorProto_has_name(const google_protobuf_ServiceDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_ServiceDescriptorProto_name(const google_protobuf_ServiceDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE const google_protobuf_MethodDescriptorProto* const* google_protobuf_ServiceDescriptorProto_method(const google_protobuf_ServiceDescriptorProto *msg, size_t *len) { return (const google_protobuf_MethodDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE bool google_protobuf_ServiceDescriptorProto_has_options(const google_protobuf_ServiceDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_ServiceOptions* google_protobuf_ServiceDescriptorProto_options(const google_protobuf_ServiceDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_ServiceOptions*, UPB_SIZE(12, 24)); } + +UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_name(google_protobuf_ServiceDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE google_protobuf_MethodDescriptorProto** google_protobuf_ServiceDescriptorProto_mutable_method(google_protobuf_ServiceDescriptorProto *msg, size_t *len) { + return (google_protobuf_MethodDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE google_protobuf_MethodDescriptorProto** google_protobuf_ServiceDescriptorProto_resize_method(google_protobuf_ServiceDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_MethodDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_MethodDescriptorProto* google_protobuf_ServiceDescriptorProto_add_method(google_protobuf_ServiceDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_MethodDescriptorProto* sub = (struct google_protobuf_MethodDescriptorProto*)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_options(google_protobuf_ServiceDescriptorProto *msg, google_protobuf_ServiceOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_ServiceOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_ServiceOptions* google_protobuf_ServiceDescriptorProto_mutable_options(google_protobuf_ServiceDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_ServiceOptions* sub = (struct google_protobuf_ServiceOptions*)google_protobuf_ServiceDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_ServiceOptions*)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_ServiceDescriptorProto_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.MethodDescriptorProto */ + +UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_MethodDescriptorProto *)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_MethodDescriptorProto *ret = google_protobuf_MethodDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_MethodDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_MethodDescriptorProto_serialize(const google_protobuf_MethodDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_MethodDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_name(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE upb_strview google_protobuf_MethodDescriptorProto_name(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_input_type(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE upb_strview google_protobuf_MethodDescriptorProto_input_type(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_output_type(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE upb_strview google_protobuf_MethodDescriptorProto_output_type(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_options(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE const google_protobuf_MethodOptions* google_protobuf_MethodDescriptorProto_options(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_MethodOptions*, UPB_SIZE(28, 56)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_client_streaming(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_client_streaming(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_server_streaming(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_server_streaming(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)); } + +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_name(google_protobuf_MethodDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_input_type(google_protobuf_MethodDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_output_type(google_protobuf_MethodDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_options(google_protobuf_MethodDescriptorProto *msg, google_protobuf_MethodOptions* value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, google_protobuf_MethodOptions*, UPB_SIZE(28, 56)) = value; +} +UPB_INLINE struct google_protobuf_MethodOptions* google_protobuf_MethodDescriptorProto_mutable_options(google_protobuf_MethodDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_MethodOptions* sub = (struct google_protobuf_MethodOptions*)google_protobuf_MethodDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_MethodOptions*)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_MethodDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_client_streaming(google_protobuf_MethodDescriptorProto *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_server_streaming(google_protobuf_MethodDescriptorProto *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)) = value; +} + +/* google.protobuf.FileOptions */ + +UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_new(upb_arena *arena) { + return (google_protobuf_FileOptions *)upb_msg_new(&google_protobuf_FileOptions_msginit, arena); +} +UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FileOptions *ret = google_protobuf_FileOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FileOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FileOptions_serialize(const google_protobuf_FileOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FileOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FileOptions_has_java_package(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 11); } +UPB_INLINE upb_strview google_protobuf_FileOptions_java_package(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(28, 32)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_outer_classname(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 12); } +UPB_INLINE upb_strview google_protobuf_FileOptions_java_outer_classname(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(36, 48)); } +UPB_INLINE bool google_protobuf_FileOptions_has_optimize_for(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_FileOptions_optimize_for(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_multiple_files(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_FileOptions_java_multiple_files(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_FileOptions_has_go_package(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 13); } +UPB_INLINE upb_strview google_protobuf_FileOptions_go_package(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(44, 64)); } +UPB_INLINE bool google_protobuf_FileOptions_has_cc_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE bool google_protobuf_FileOptions_cc_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(17, 17)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE bool google_protobuf_FileOptions_java_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(18, 18)); } +UPB_INLINE bool google_protobuf_FileOptions_has_py_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE bool google_protobuf_FileOptions_py_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(19, 19)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_generate_equals_and_hash(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE bool google_protobuf_FileOptions_java_generate_equals_and_hash(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(20, 20)); } +UPB_INLINE bool google_protobuf_FileOptions_has_deprecated(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 7); } +UPB_INLINE bool google_protobuf_FileOptions_deprecated(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(21, 21)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_string_check_utf8(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 8); } +UPB_INLINE bool google_protobuf_FileOptions_java_string_check_utf8(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(22, 22)); } +UPB_INLINE bool google_protobuf_FileOptions_has_cc_enable_arenas(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 9); } +UPB_INLINE bool google_protobuf_FileOptions_cc_enable_arenas(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(23, 23)); } +UPB_INLINE bool google_protobuf_FileOptions_has_objc_class_prefix(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 14); } +UPB_INLINE upb_strview google_protobuf_FileOptions_objc_class_prefix(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(52, 80)); } +UPB_INLINE bool google_protobuf_FileOptions_has_csharp_namespace(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 15); } +UPB_INLINE upb_strview google_protobuf_FileOptions_csharp_namespace(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(60, 96)); } +UPB_INLINE bool google_protobuf_FileOptions_has_swift_prefix(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 16); } +UPB_INLINE upb_strview google_protobuf_FileOptions_swift_prefix(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(68, 112)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_class_prefix(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 17); } +UPB_INLINE upb_strview google_protobuf_FileOptions_php_class_prefix(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(76, 128)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_namespace(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 18); } +UPB_INLINE upb_strview google_protobuf_FileOptions_php_namespace(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(84, 144)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 10); } +UPB_INLINE bool google_protobuf_FileOptions_php_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_metadata_namespace(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 19); } +UPB_INLINE upb_strview google_protobuf_FileOptions_php_metadata_namespace(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(92, 160)); } +UPB_INLINE bool google_protobuf_FileOptions_has_ruby_package(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 20); } +UPB_INLINE upb_strview google_protobuf_FileOptions_ruby_package(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(100, 176)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_FileOptions_uninterpreted_option(const google_protobuf_FileOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(108, 192), len); } + +UPB_INLINE void google_protobuf_FileOptions_set_java_package(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 11); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(28, 32)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_outer_classname(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 12); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(36, 48)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_optimize_for(google_protobuf_FileOptions *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_multiple_files(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_go_package(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 13); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(44, 64)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_cc_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, bool, UPB_SIZE(17, 17)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, bool, UPB_SIZE(18, 18)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_py_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, bool, UPB_SIZE(19, 19)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_generate_equals_and_hash(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, bool, UPB_SIZE(20, 20)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_deprecated(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 7); + UPB_FIELD_AT(msg, bool, UPB_SIZE(21, 21)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_string_check_utf8(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 8); + UPB_FIELD_AT(msg, bool, UPB_SIZE(22, 22)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_cc_enable_arenas(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 9); + UPB_FIELD_AT(msg, bool, UPB_SIZE(23, 23)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_objc_class_prefix(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 14); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(52, 80)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_csharp_namespace(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 15); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(60, 96)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_swift_prefix(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 16); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(68, 112)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_class_prefix(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 17); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(76, 128)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_namespace(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 18); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(84, 144)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 10); + UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_metadata_namespace(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 19); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(92, 160)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_ruby_package(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 20); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(100, 176)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FileOptions_mutable_uninterpreted_option(google_protobuf_FileOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(108, 192), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FileOptions_resize_uninterpreted_option(google_protobuf_FileOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(108, 192), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_FileOptions_add_uninterpreted_option(google_protobuf_FileOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(108, 192), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.MessageOptions */ + +UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_new(upb_arena *arena) { + return (google_protobuf_MessageOptions *)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); +} +UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_MessageOptions *ret = google_protobuf_MessageOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_MessageOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_MessageOptions_serialize(const google_protobuf_MessageOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_MessageOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_MessageOptions_has_message_set_wire_format(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_MessageOptions_message_set_wire_format(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE bool google_protobuf_MessageOptions_has_no_standard_descriptor_accessor(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_MessageOptions_no_standard_descriptor_accessor(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)); } +UPB_INLINE bool google_protobuf_MessageOptions_has_deprecated(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE bool google_protobuf_MessageOptions_deprecated(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(3, 3)); } +UPB_INLINE bool google_protobuf_MessageOptions_has_map_entry(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE bool google_protobuf_MessageOptions_map_entry(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(4, 4)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_MessageOptions_uninterpreted_option(const google_protobuf_MessageOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(8, 8), len); } + +UPB_INLINE void google_protobuf_MessageOptions_set_message_set_wire_format(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE void google_protobuf_MessageOptions_set_no_standard_descriptor_accessor(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)) = value; +} +UPB_INLINE void google_protobuf_MessageOptions_set_deprecated(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, bool, UPB_SIZE(3, 3)) = value; +} +UPB_INLINE void google_protobuf_MessageOptions_set_map_entry(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, bool, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MessageOptions_mutable_uninterpreted_option(google_protobuf_MessageOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(8, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MessageOptions_resize_uninterpreted_option(google_protobuf_MessageOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(8, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_MessageOptions_add_uninterpreted_option(google_protobuf_MessageOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(8, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.FieldOptions */ + +UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_new(upb_arena *arena) { + return (google_protobuf_FieldOptions *)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); +} +UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FieldOptions *ret = google_protobuf_FieldOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FieldOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FieldOptions_serialize(const google_protobuf_FieldOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FieldOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FieldOptions_has_ctype(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_FieldOptions_ctype(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_packed(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE bool google_protobuf_FieldOptions_packed(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_deprecated(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE bool google_protobuf_FieldOptions_deprecated(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(25, 25)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_lazy(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE bool google_protobuf_FieldOptions_lazy(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(26, 26)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_jstype(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_FieldOptions_jstype(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_weak(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE bool google_protobuf_FieldOptions_weak(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(27, 27)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_FieldOptions_uninterpreted_option(const google_protobuf_FieldOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(28, 32), len); } + +UPB_INLINE void google_protobuf_FieldOptions_set_ctype(google_protobuf_FieldOptions *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_packed(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_deprecated(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, bool, UPB_SIZE(25, 25)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_lazy(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, bool, UPB_SIZE(26, 26)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_jstype(google_protobuf_FieldOptions *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_weak(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, bool, UPB_SIZE(27, 27)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FieldOptions_mutable_uninterpreted_option(google_protobuf_FieldOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(28, 32), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FieldOptions_resize_uninterpreted_option(google_protobuf_FieldOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(28, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_FieldOptions_add_uninterpreted_option(google_protobuf_FieldOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(28, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.OneofOptions */ + +UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_new(upb_arena *arena) { + return (google_protobuf_OneofOptions *)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); +} +UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_OneofOptions *ret = google_protobuf_OneofOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_OneofOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_OneofOptions_serialize(const google_protobuf_OneofOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_OneofOptions_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_OneofOptions_uninterpreted_option(const google_protobuf_OneofOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_OneofOptions_mutable_uninterpreted_option(google_protobuf_OneofOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_OneofOptions_resize_uninterpreted_option(google_protobuf_OneofOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_OneofOptions_add_uninterpreted_option(google_protobuf_OneofOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.EnumOptions */ + +UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_new(upb_arena *arena) { + return (google_protobuf_EnumOptions *)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); +} +UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumOptions *ret = google_protobuf_EnumOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumOptions_serialize(const google_protobuf_EnumOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumOptions_has_allow_alias(const google_protobuf_EnumOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_EnumOptions_allow_alias(const google_protobuf_EnumOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE bool google_protobuf_EnumOptions_has_deprecated(const google_protobuf_EnumOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_EnumOptions_deprecated(const google_protobuf_EnumOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_EnumOptions_uninterpreted_option(const google_protobuf_EnumOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } + +UPB_INLINE void google_protobuf_EnumOptions_set_allow_alias(google_protobuf_EnumOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE void google_protobuf_EnumOptions_set_deprecated(google_protobuf_EnumOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumOptions_mutable_uninterpreted_option(google_protobuf_EnumOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumOptions_resize_uninterpreted_option(google_protobuf_EnumOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_EnumOptions_add_uninterpreted_option(google_protobuf_EnumOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.EnumValueOptions */ + +UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_new(upb_arena *arena) { + return (google_protobuf_EnumValueOptions *)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); +} +UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumValueOptions *ret = google_protobuf_EnumValueOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumValueOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumValueOptions_serialize(const google_protobuf_EnumValueOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumValueOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumValueOptions_has_deprecated(const google_protobuf_EnumValueOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_EnumValueOptions_deprecated(const google_protobuf_EnumValueOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_EnumValueOptions_uninterpreted_option(const google_protobuf_EnumValueOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } + +UPB_INLINE void google_protobuf_EnumValueOptions_set_deprecated(google_protobuf_EnumValueOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumValueOptions_mutable_uninterpreted_option(google_protobuf_EnumValueOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumValueOptions_resize_uninterpreted_option(google_protobuf_EnumValueOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_EnumValueOptions_add_uninterpreted_option(google_protobuf_EnumValueOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.ServiceOptions */ + +UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_new(upb_arena *arena) { + return (google_protobuf_ServiceOptions *)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); +} +UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_ServiceOptions *ret = google_protobuf_ServiceOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_ServiceOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_ServiceOptions_serialize(const google_protobuf_ServiceOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_ServiceOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_ServiceOptions_has_deprecated(const google_protobuf_ServiceOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_ServiceOptions_deprecated(const google_protobuf_ServiceOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_ServiceOptions_uninterpreted_option(const google_protobuf_ServiceOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } + +UPB_INLINE void google_protobuf_ServiceOptions_set_deprecated(google_protobuf_ServiceOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ServiceOptions_mutable_uninterpreted_option(google_protobuf_ServiceOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ServiceOptions_resize_uninterpreted_option(google_protobuf_ServiceOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_ServiceOptions_add_uninterpreted_option(google_protobuf_ServiceOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.MethodOptions */ + +UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_new(upb_arena *arena) { + return (google_protobuf_MethodOptions *)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); +} +UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_MethodOptions *ret = google_protobuf_MethodOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_MethodOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_MethodOptions_serialize(const google_protobuf_MethodOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_MethodOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_MethodOptions_has_deprecated(const google_protobuf_MethodOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_MethodOptions_deprecated(const google_protobuf_MethodOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_MethodOptions_has_idempotency_level(const google_protobuf_MethodOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_MethodOptions_idempotency_level(const google_protobuf_MethodOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_MethodOptions_uninterpreted_option(const google_protobuf_MethodOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(20, 24), len); } + +UPB_INLINE void google_protobuf_MethodOptions_set_deprecated(google_protobuf_MethodOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_MethodOptions_set_idempotency_level(google_protobuf_MethodOptions *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MethodOptions_mutable_uninterpreted_option(google_protobuf_MethodOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 24), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MethodOptions_resize_uninterpreted_option(google_protobuf_MethodOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 24), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_MethodOptions_add_uninterpreted_option(google_protobuf_MethodOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 24), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.UninterpretedOption */ + +UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_new(upb_arena *arena) { + return (google_protobuf_UninterpretedOption *)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); +} +UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_UninterpretedOption *ret = google_protobuf_UninterpretedOption_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_UninterpretedOption_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_UninterpretedOption_serialize(const google_protobuf_UninterpretedOption *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_UninterpretedOption_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_UninterpretedOption_NamePart* const* google_protobuf_UninterpretedOption_name(const google_protobuf_UninterpretedOption *msg, size_t *len) { return (const google_protobuf_UninterpretedOption_NamePart* const*)_upb_array_accessor(msg, UPB_SIZE(56, 80), len); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_identifier_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_identifier_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_positive_int_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE uint64_t google_protobuf_UninterpretedOption_positive_int_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, uint64_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_negative_int_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int64_t google_protobuf_UninterpretedOption_negative_int_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, int64_t, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_double_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE double google_protobuf_UninterpretedOption_double_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, double, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_string_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_string_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_aggregate_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_aggregate_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)); } + +UPB_INLINE google_protobuf_UninterpretedOption_NamePart** google_protobuf_UninterpretedOption_mutable_name(google_protobuf_UninterpretedOption *msg, size_t *len) { + return (google_protobuf_UninterpretedOption_NamePart**)_upb_array_mutable_accessor(msg, UPB_SIZE(56, 80), len); +} +UPB_INLINE google_protobuf_UninterpretedOption_NamePart** google_protobuf_UninterpretedOption_resize_name(google_protobuf_UninterpretedOption *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption_NamePart**)_upb_array_resize_accessor(msg, UPB_SIZE(56, 80), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption_NamePart* google_protobuf_UninterpretedOption_add_name(google_protobuf_UninterpretedOption *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption_NamePart* sub = (struct google_protobuf_UninterpretedOption_NamePart*)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(56, 80), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_identifier_value(google_protobuf_UninterpretedOption *msg, upb_strview value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_positive_int_value(google_protobuf_UninterpretedOption *msg, uint64_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, uint64_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_negative_int_value(google_protobuf_UninterpretedOption *msg, int64_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int64_t, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_double_value(google_protobuf_UninterpretedOption *msg, double value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, double, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_string_value(google_protobuf_UninterpretedOption *msg, upb_strview value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_aggregate_value(google_protobuf_UninterpretedOption *msg, upb_strview value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)) = value; +} + +/* google.protobuf.UninterpretedOption.NamePart */ + +UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_new(upb_arena *arena) { + return (google_protobuf_UninterpretedOption_NamePart *)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); +} +UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_UninterpretedOption_NamePart *ret = google_protobuf_UninterpretedOption_NamePart_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_UninterpretedOption_NamePart_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_UninterpretedOption_NamePart_serialize(const google_protobuf_UninterpretedOption_NamePart *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_UninterpretedOption_NamePart_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_UninterpretedOption_NamePart_has_name_part(const google_protobuf_UninterpretedOption_NamePart *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_NamePart_name_part(const google_protobuf_UninterpretedOption_NamePart *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_NamePart_has_is_extension(const google_protobuf_UninterpretedOption_NamePart *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_UninterpretedOption_NamePart_is_extension(const google_protobuf_UninterpretedOption_NamePart *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } + +UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_name_part(google_protobuf_UninterpretedOption_NamePart *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_is_extension(google_protobuf_UninterpretedOption_NamePart *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} + +/* google.protobuf.SourceCodeInfo */ + +UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_new(upb_arena *arena) { + return (google_protobuf_SourceCodeInfo *)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); +} +UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_SourceCodeInfo *ret = google_protobuf_SourceCodeInfo_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_SourceCodeInfo_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_SourceCodeInfo_serialize(const google_protobuf_SourceCodeInfo *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_SourceCodeInfo_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_SourceCodeInfo_Location* const* google_protobuf_SourceCodeInfo_location(const google_protobuf_SourceCodeInfo *msg, size_t *len) { return (const google_protobuf_SourceCodeInfo_Location* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_SourceCodeInfo_Location** google_protobuf_SourceCodeInfo_mutable_location(google_protobuf_SourceCodeInfo *msg, size_t *len) { + return (google_protobuf_SourceCodeInfo_Location**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_SourceCodeInfo_Location** google_protobuf_SourceCodeInfo_resize_location(google_protobuf_SourceCodeInfo *msg, size_t len, upb_arena *arena) { + return (google_protobuf_SourceCodeInfo_Location**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_SourceCodeInfo_Location* google_protobuf_SourceCodeInfo_add_location(google_protobuf_SourceCodeInfo *msg, upb_arena *arena) { + struct google_protobuf_SourceCodeInfo_Location* sub = (struct google_protobuf_SourceCodeInfo_Location*)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.SourceCodeInfo.Location */ + +UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_new(upb_arena *arena) { + return (google_protobuf_SourceCodeInfo_Location *)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); +} +UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_SourceCodeInfo_Location *ret = google_protobuf_SourceCodeInfo_Location_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_SourceCodeInfo_Location_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_SourceCodeInfo_Location_serialize(const google_protobuf_SourceCodeInfo_Location *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_SourceCodeInfo_Location_msginit, arena, len); +} + +UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_path(const google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } +UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_span(const google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(24, 48), len); } +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_leading_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_SourceCodeInfo_Location_leading_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_trailing_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_SourceCodeInfo_Location_trailing_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)); } +UPB_INLINE upb_strview const* google_protobuf_SourceCodeInfo_Location_leading_detached_comments(const google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(28, 56), len); } + +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_mutable_path(google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_resize_path(google_protobuf_SourceCodeInfo_Location *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_add_path(google_protobuf_SourceCodeInfo_Location *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_mutable_span(google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(24, 48), len); +} +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_resize_span(google_protobuf_SourceCodeInfo_Location *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(24, 48), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_add_span(google_protobuf_SourceCodeInfo_Location *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(24, 48), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_leading_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_trailing_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE upb_strview* google_protobuf_SourceCodeInfo_Location_mutable_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(28, 56), len); +} +UPB_INLINE upb_strview* google_protobuf_SourceCodeInfo_Location_resize_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(28, 56), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_add_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(28, 56), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} + +/* google.protobuf.GeneratedCodeInfo */ + +UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_new(upb_arena *arena) { + return (google_protobuf_GeneratedCodeInfo *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena); +} +UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_GeneratedCodeInfo *ret = google_protobuf_GeneratedCodeInfo_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_GeneratedCodeInfo_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_GeneratedCodeInfo_serialize(const google_protobuf_GeneratedCodeInfo *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_GeneratedCodeInfo_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_GeneratedCodeInfo_Annotation* const* google_protobuf_GeneratedCodeInfo_annotation(const google_protobuf_GeneratedCodeInfo *msg, size_t *len) { return (const google_protobuf_GeneratedCodeInfo_Annotation* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation** google_protobuf_GeneratedCodeInfo_mutable_annotation(google_protobuf_GeneratedCodeInfo *msg, size_t *len) { + return (google_protobuf_GeneratedCodeInfo_Annotation**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation** google_protobuf_GeneratedCodeInfo_resize_annotation(google_protobuf_GeneratedCodeInfo *msg, size_t len, upb_arena *arena) { + return (google_protobuf_GeneratedCodeInfo_Annotation**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_GeneratedCodeInfo_Annotation* google_protobuf_GeneratedCodeInfo_add_annotation(google_protobuf_GeneratedCodeInfo *msg, upb_arena *arena) { + struct google_protobuf_GeneratedCodeInfo_Annotation* sub = (struct google_protobuf_GeneratedCodeInfo_Annotation*)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.GeneratedCodeInfo.Annotation */ + +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_new(upb_arena *arena) { + return (google_protobuf_GeneratedCodeInfo_Annotation *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); +} +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_GeneratedCodeInfo_Annotation *ret = google_protobuf_GeneratedCodeInfo_Annotation_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_GeneratedCodeInfo_Annotation_serialize(const google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena, len); +} + +UPB_INLINE int32_t const* google_protobuf_GeneratedCodeInfo_Annotation_path(const google_protobuf_GeneratedCodeInfo_Annotation *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(20, 32), len); } +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_source_file(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE upb_strview google_protobuf_GeneratedCodeInfo_Annotation_source_file(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 16)); } +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_begin(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_GeneratedCodeInfo_Annotation_begin(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_end(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_GeneratedCodeInfo_Annotation_end(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE int32_t* google_protobuf_GeneratedCodeInfo_Annotation_mutable_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 32), len); +} +UPB_INLINE int32_t* google_protobuf_GeneratedCodeInfo_Annotation_resize_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(20, 32), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_add_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(20, 32), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE void google_protobuf_GeneratedCodeInfo_Annotation_set_source_file(google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_strview value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 16)) = value; +} +UPB_INLINE void google_protobuf_GeneratedCodeInfo_Annotation_set_begin(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_GeneratedCodeInfo_Annotation_set_end(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ */ diff --git a/third_party/upb/generated_for_cmake/upb/json/parser.c b/third_party/upb/generated_for_cmake/upb/json/parser.c new file mode 100644 index 00000000000..f72e945881f --- /dev/null +++ b/third_party/upb/generated_for_cmake/upb/json/parser.c @@ -0,0 +1,3454 @@ + +#line 1 "upb/json/parser.rl" +/* +** upb::json::Parser (upb_json_parser) +** +** A parser that uses the Ragel State Machine Compiler to generate +** the finite automata. +** +** Ragel only natively handles regular languages, but we can manually +** program it a bit to handle context-free languages like JSON, by using +** the "fcall" and "fret" constructs. +** +** This parser can handle the basics, but needs several things to be fleshed +** out: +** +** - handling of unicode escape sequences (including high surrogate pairs). +** - properly check and report errors for unknown fields, stack overflow, +** improper array nesting (or lack of nesting). +** - handling of base64 sequences with padding characters. +** - handling of push-back (non-success returns from sink functions). +** - handling of keys/escape-sequences/etc that span input buffers. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "upb/json/parser.h" +#include "upb/pb/encoder.h" + +#include "upb/port_def.inc" + +#define UPB_JSON_MAX_DEPTH 64 + +/* Type of value message */ +enum { + VALUE_NULLVALUE = 0, + VALUE_NUMBERVALUE = 1, + VALUE_STRINGVALUE = 2, + VALUE_BOOLVALUE = 3, + VALUE_STRUCTVALUE = 4, + VALUE_LISTVALUE = 5 +}; + +/* Forward declare */ +static bool is_top_level(upb_json_parser *p); +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type); +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type); + +static bool is_number_wrapper_object(upb_json_parser *p); +static bool does_number_wrapper_start(upb_json_parser *p); +static bool does_number_wrapper_end(upb_json_parser *p); + +static bool is_string_wrapper_object(upb_json_parser *p); +static bool does_string_wrapper_start(upb_json_parser *p); +static bool does_string_wrapper_end(upb_json_parser *p); + +static bool does_fieldmask_start(upb_json_parser *p); +static bool does_fieldmask_end(upb_json_parser *p); +static void start_fieldmask_object(upb_json_parser *p); +static void end_fieldmask_object(upb_json_parser *p); + +static void start_wrapper_object(upb_json_parser *p); +static void end_wrapper_object(upb_json_parser *p); + +static void start_value_object(upb_json_parser *p, int value_type); +static void end_value_object(upb_json_parser *p); + +static void start_listvalue_object(upb_json_parser *p); +static void end_listvalue_object(upb_json_parser *p); + +static void start_structvalue_object(upb_json_parser *p); +static void end_structvalue_object(upb_json_parser *p); + +static void start_object(upb_json_parser *p); +static void end_object(upb_json_parser *p); + +static void start_any_object(upb_json_parser *p, const char *ptr); +static bool end_any_object(upb_json_parser *p, const char *ptr); + +static bool start_subobject(upb_json_parser *p); +static void end_subobject(upb_json_parser *p); + +static void start_member(upb_json_parser *p); +static void end_member(upb_json_parser *p); +static bool end_membername(upb_json_parser *p); + +static void start_any_member(upb_json_parser *p, const char *ptr); +static void end_any_member(upb_json_parser *p, const char *ptr); +static bool end_any_membername(upb_json_parser *p); + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle); +static bool end(void *closure, const void *hd); + +static const char eof_ch = 'e'; + +/* stringsink */ +typedef struct { + upb_byteshandler handler; + upb_bytessink sink; + char *ptr; + size_t len, size; +} upb_stringsink; + + +static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) { + upb_stringsink *sink = _sink; + sink->len = 0; + UPB_UNUSED(hd); + UPB_UNUSED(size_hint); + return sink; +} + +static size_t stringsink_string(void *_sink, const void *hd, const char *ptr, + size_t len, const upb_bufhandle *handle) { + upb_stringsink *sink = _sink; + size_t new_size = sink->size; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + while (sink->len + len > new_size) { + new_size *= 2; + } + + if (new_size != sink->size) { + sink->ptr = realloc(sink->ptr, new_size); + sink->size = new_size; + } + + memcpy(sink->ptr + sink->len, ptr, len); + sink->len += len; + + return len; +} + +void upb_stringsink_init(upb_stringsink *sink) { + upb_byteshandler_init(&sink->handler); + upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL); + upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL); + + upb_bytessink_reset(&sink->sink, &sink->handler, sink); + + sink->size = 32; + sink->ptr = malloc(sink->size); + sink->len = 0; +} + +void upb_stringsink_uninit(upb_stringsink *sink) { free(sink->ptr); } + +typedef struct { + /* For encoding Any value field in binary format. */ + upb_handlercache *encoder_handlercache; + upb_stringsink stringsink; + + /* For decoding Any value field in json format. */ + upb_json_codecache *parser_codecache; + upb_sink sink; + upb_json_parser *parser; + + /* Mark the range of uninterpreted values in json input before type url. */ + const char *before_type_url_start; + const char *before_type_url_end; + + /* Mark the range of uninterpreted values in json input after type url. */ + const char *after_type_url_start; +} upb_jsonparser_any_frame; + +typedef struct { + upb_sink sink; + + /* The current message in which we're parsing, and the field whose value we're + * expecting next. */ + const upb_msgdef *m; + const upb_fielddef *f; + + /* The table mapping json name to fielddef for this message. */ + const upb_strtable *name_table; + + /* We are in a repeated-field context. We need this flag to decide whether to + * handle the array as a normal repeated field or a + * google.protobuf.ListValue/google.protobuf.Value. */ + bool is_repeated; + + /* We are in a repeated-field context, ready to emit mapentries as + * submessages. This flag alters the start-of-object (open-brace) behavior to + * begin a sequence of mapentry messages rather than a single submessage. */ + bool is_map; + + /* We are in a map-entry message context. This flag is set when parsing the + * value field of a single map entry and indicates to all value-field parsers + * (subobjects, strings, numbers, and bools) that the map-entry submessage + * should end as soon as the value is parsed. */ + bool is_mapentry; + + /* If |is_map| or |is_mapentry| is true, |mapfield| refers to the parent + * message's map field that we're currently parsing. This differs from |f| + * because |f| is the field in the *current* message (i.e., the map-entry + * message itself), not the parent's field that leads to this map. */ + const upb_fielddef *mapfield; + + /* We are in an Any message context. This flag is set when parsing the Any + * message and indicates to all field parsers (subobjects, strings, numbers, + * and bools) that the parsed field should be serialized as binary data or + * cached (type url not found yet). */ + bool is_any; + + /* The type of packed message in Any. */ + upb_jsonparser_any_frame *any_frame; + + /* True if the field to be parsed is unknown. */ + bool is_unknown_field; +} upb_jsonparser_frame; + +static void init_frame(upb_jsonparser_frame* frame) { + frame->m = NULL; + frame->f = NULL; + frame->name_table = NULL; + frame->is_repeated = false; + frame->is_map = false; + frame->is_mapentry = false; + frame->mapfield = NULL; + frame->is_any = false; + frame->any_frame = NULL; + frame->is_unknown_field = false; +} + +struct upb_json_parser { + upb_arena *arena; + const upb_json_parsermethod *method; + upb_bytessink input_; + + /* Stack to track the JSON scopes we are in. */ + upb_jsonparser_frame stack[UPB_JSON_MAX_DEPTH]; + upb_jsonparser_frame *top; + upb_jsonparser_frame *limit; + + upb_status *status; + + /* Ragel's internal parsing stack for the parsing state machine. */ + int current_state; + int parser_stack[UPB_JSON_MAX_DEPTH]; + int parser_top; + + /* The handle for the current buffer. */ + const upb_bufhandle *handle; + + /* Accumulate buffer. See details in parser.rl. */ + const char *accumulated; + size_t accumulated_len; + char *accumulate_buf; + size_t accumulate_buf_size; + + /* Multi-part text data. See details in parser.rl. */ + int multipart_state; + upb_selector_t string_selector; + + /* Input capture. See details in parser.rl. */ + const char *capture; + + /* Intermediate result of parsing a unicode escape sequence. */ + uint32_t digit; + + /* For resolve type url in Any. */ + const upb_symtab *symtab; + + /* Whether to proceed if unknown field is met. */ + bool ignore_json_unknown; + + /* Cache for parsing timestamp due to base and zone are handled in different + * handlers. */ + struct tm tm; +}; + +static upb_jsonparser_frame* start_jsonparser_frame(upb_json_parser *p) { + upb_jsonparser_frame *inner; + inner = p->top + 1; + init_frame(inner); + return inner; +} + +struct upb_json_codecache { + upb_arena *arena; + upb_inttable methods; /* upb_msgdef* -> upb_json_parsermethod* */ +}; + +struct upb_json_parsermethod { + const upb_json_codecache *cache; + upb_byteshandler input_handler_; + + /* Maps json_name -> fielddef */ + upb_strtable name_table; +}; + +#define PARSER_CHECK_RETURN(x) if (!(x)) return false + +static upb_jsonparser_any_frame *json_parser_any_frame_new( + upb_json_parser *p) { + upb_jsonparser_any_frame *frame; + + frame = upb_arena_malloc(p->arena, sizeof(upb_jsonparser_any_frame)); + + frame->encoder_handlercache = upb_pb_encoder_newcache(); + frame->parser_codecache = upb_json_codecache_new(); + frame->parser = NULL; + frame->before_type_url_start = NULL; + frame->before_type_url_end = NULL; + frame->after_type_url_start = NULL; + + upb_stringsink_init(&frame->stringsink); + + return frame; +} + +static void json_parser_any_frame_set_payload_type( + upb_json_parser *p, + upb_jsonparser_any_frame *frame, + const upb_msgdef *payload_type) { + const upb_handlers *h; + const upb_json_parsermethod *parser_method; + upb_pb_encoder *encoder; + + /* Initialize encoder. */ + h = upb_handlercache_get(frame->encoder_handlercache, payload_type); + encoder = upb_pb_encoder_create(p->arena, h, frame->stringsink.sink); + + /* Initialize parser. */ + parser_method = upb_json_codecache_get(frame->parser_codecache, payload_type); + upb_sink_reset(&frame->sink, h, encoder); + frame->parser = + upb_json_parser_create(p->arena, parser_method, p->symtab, frame->sink, + p->status, p->ignore_json_unknown); +} + +static void json_parser_any_frame_free(upb_jsonparser_any_frame *frame) { + upb_handlercache_free(frame->encoder_handlercache); + upb_json_codecache_free(frame->parser_codecache); + upb_stringsink_uninit(&frame->stringsink); +} + +static bool json_parser_any_frame_has_type_url( + upb_jsonparser_any_frame *frame) { + return frame->parser != NULL; +} + +static bool json_parser_any_frame_has_value_before_type_url( + upb_jsonparser_any_frame *frame) { + return frame->before_type_url_start != frame->before_type_url_end; +} + +static bool json_parser_any_frame_has_value_after_type_url( + upb_jsonparser_any_frame *frame) { + return frame->after_type_url_start != NULL; +} + +static bool json_parser_any_frame_has_value( + upb_jsonparser_any_frame *frame) { + return json_parser_any_frame_has_value_before_type_url(frame) || + json_parser_any_frame_has_value_after_type_url(frame); +} + +static void json_parser_any_frame_set_before_type_url_end( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (frame->parser == NULL) { + frame->before_type_url_end = ptr; + } +} + +static void json_parser_any_frame_set_after_type_url_start_once( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (json_parser_any_frame_has_type_url(frame) && + frame->after_type_url_start == NULL) { + frame->after_type_url_start = ptr; + } +} + +/* Used to signal that a capture has been suspended. */ +static char suspend_capture; + +static upb_selector_t getsel_for_handlertype(upb_json_parser *p, + upb_handlertype_t type) { + upb_selector_t sel; + bool ok = upb_handlers_getselector(p->top->f, type, &sel); + UPB_ASSERT(ok); + return sel; +} + +static upb_selector_t parser_getsel(upb_json_parser *p) { + return getsel_for_handlertype( + p, upb_handlers_getprimitivehandlertype(p->top->f)); +} + +static bool check_stack(upb_json_parser *p) { + if ((p->top + 1) == p->limit) { + upb_status_seterrmsg(p->status, "Nesting too deep"); + return false; + } + + return true; +} + +static void set_name_table(upb_json_parser *p, upb_jsonparser_frame *frame) { + upb_value v; + const upb_json_codecache *cache = p->method->cache; + bool ok; + const upb_json_parsermethod *method; + + ok = upb_inttable_lookupptr(&cache->methods, frame->m, &v); + UPB_ASSERT(ok); + method = upb_value_getconstptr(v); + + frame->name_table = &method->name_table; +} + +/* There are GCC/Clang built-ins for overflow checking which we could start + * using if there was any performance benefit to it. */ + +static bool checked_add(size_t a, size_t b, size_t *c) { + if (SIZE_MAX - a < b) return false; + *c = a + b; + return true; +} + +static size_t saturating_multiply(size_t a, size_t b) { + /* size_t is unsigned, so this is defined behavior even on overflow. */ + size_t ret = a * b; + if (b != 0 && ret / b != a) { + ret = SIZE_MAX; + } + return ret; +} + + +/* Base64 decoding ************************************************************/ + +/* TODO(haberman): make this streaming. */ + +static const signed char b64table[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62/*+*/, -1, -1, -1, 63/*/ */, + 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, + 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, + -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, + 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, + 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1, + -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, + 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/, + 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/, + 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* Returns the table value sign-extended to 32 bits. Knowing that the upper + * bits will be 1 for unrecognized characters makes it easier to check for + * this error condition later (see below). */ +int32_t b64lookup(unsigned char ch) { return b64table[ch]; } + +/* Returns true if the given character is not a valid base64 character or + * padding. */ +bool nonbase64(unsigned char ch) { return b64lookup(ch) == -1 && ch != '='; } + +static bool base64_push(upb_json_parser *p, upb_selector_t sel, const char *ptr, + size_t len) { + const char *limit = ptr + len; + for (; ptr < limit; ptr += 4) { + uint32_t val; + char output[3]; + + if (limit - ptr < 4) { + upb_status_seterrf(p->status, + "Base64 input for bytes field not a multiple of 4: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6 | + b64lookup(ptr[3]); + + /* Test the upper bit; returns true if any of the characters returned -1. */ + if (val & 0x80000000) { + goto otherchar; + } + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + output[2] = val & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 3, NULL); + } + return true; + +otherchar: + if (nonbase64(ptr[0]) || nonbase64(ptr[1]) || nonbase64(ptr[2]) || + nonbase64(ptr[3]) ) { + upb_status_seterrf(p->status, + "Non-base64 characters in bytes field: %s", + upb_fielddef_name(p->top->f)); + return false; + } if (ptr[2] == '=') { + uint32_t val; + char output; + + /* Last group contains only two input bytes, one output byte. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[3] != '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12; + + UPB_ASSERT(!(val & 0x80000000)); + output = val >> 16; + upb_sink_putstring(p->top->sink, sel, &output, 1, NULL); + return true; + } else { + uint32_t val; + char output[2]; + + /* Last group contains only three input bytes, two output bytes. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[2] == '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6; + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 2, NULL); + return true; + } + +badpadding: + upb_status_seterrf(p->status, + "Incorrect base64 padding for field: %s (%.*s)", + upb_fielddef_name(p->top->f), + 4, ptr); + return false; +} + + +/* Accumulate buffer **********************************************************/ + +/* Functionality for accumulating a buffer. + * + * Some parts of the parser need an entire value as a contiguous string. For + * example, to look up a member name in a hash table, or to turn a string into + * a number, the relevant library routines need the input string to be in + * contiguous memory, even if the value spanned two or more buffers in the + * input. These routines handle that. + * + * In the common case we can just point to the input buffer to get this + * contiguous string and avoid any actual copy. So we optimistically begin + * this way. But there are a few cases where we must instead copy into a + * separate buffer: + * + * 1. The string was not contiguous in the input (it spanned buffers). + * + * 2. The string included escape sequences that need to be interpreted to get + * the true value in a contiguous buffer. */ + +static void assert_accumulate_empty(upb_json_parser *p) { + UPB_ASSERT(p->accumulated == NULL); + UPB_ASSERT(p->accumulated_len == 0); +} + +static void accumulate_clear(upb_json_parser *p) { + p->accumulated = NULL; + p->accumulated_len = 0; +} + +/* Used internally by accumulate_append(). */ +static bool accumulate_realloc(upb_json_parser *p, size_t need) { + void *mem; + size_t old_size = p->accumulate_buf_size; + size_t new_size = UPB_MAX(old_size, 128); + while (new_size < need) { + new_size = saturating_multiply(new_size, 2); + } + + mem = upb_arena_realloc(p->arena, p->accumulate_buf, old_size, new_size); + if (!mem) { + upb_status_seterrmsg(p->status, "Out of memory allocating buffer."); + return false; + } + + p->accumulate_buf = mem; + p->accumulate_buf_size = new_size; + return true; +} + +/* Logically appends the given data to the append buffer. + * If "can_alias" is true, we will try to avoid actually copying, but the buffer + * must be valid until the next accumulate_append() call (if any). */ +static bool accumulate_append(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + size_t need; + + if (!p->accumulated && can_alias) { + p->accumulated = buf; + p->accumulated_len = len; + return true; + } + + if (!checked_add(p->accumulated_len, len, &need)) { + upb_status_seterrmsg(p->status, "Integer overflow."); + return false; + } + + if (need > p->accumulate_buf_size && !accumulate_realloc(p, need)) { + return false; + } + + if (p->accumulated != p->accumulate_buf) { + memcpy(p->accumulate_buf, p->accumulated, p->accumulated_len); + p->accumulated = p->accumulate_buf; + } + + memcpy(p->accumulate_buf + p->accumulated_len, buf, len); + p->accumulated_len += len; + return true; +} + +/* Returns a pointer to the data accumulated since the last accumulate_clear() + * call, and writes the length to *len. This with point either to the input + * buffer or a temporary accumulate buffer. */ +static const char *accumulate_getptr(upb_json_parser *p, size_t *len) { + UPB_ASSERT(p->accumulated); + *len = p->accumulated_len; + return p->accumulated; +} + + +/* Mult-part text data ********************************************************/ + +/* When we have text data in the input, it can often come in multiple segments. + * For example, there may be some raw string data followed by an escape + * sequence. The two segments are processed with different logic. Also buffer + * seams in the input can cause multiple segments. + * + * As we see segments, there are two main cases for how we want to process them: + * + * 1. we want to push the captured input directly to string handlers. + * + * 2. we need to accumulate all the parts into a contiguous buffer for further + * processing (field name lookup, string->number conversion, etc). */ + +/* This is the set of states for p->multipart_state. */ +enum { + /* We are not currently processing multipart data. */ + MULTIPART_INACTIVE = 0, + + /* We are processing multipart data by accumulating it into a contiguous + * buffer. */ + MULTIPART_ACCUMULATE = 1, + + /* We are processing multipart data by pushing each part directly to the + * current string handlers. */ + MULTIPART_PUSHEAGERLY = 2 +}; + +/* Start a multi-part text value where we accumulate the data for processing at + * the end. */ +static void multipart_startaccum(upb_json_parser *p) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_ACCUMULATE; +} + +/* Start a multi-part text value where we immediately push text data to a string + * value with the given selector. */ +static void multipart_start(upb_json_parser *p, upb_selector_t sel) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_PUSHEAGERLY; + p->string_selector = sel; +} + +static bool multipart_text(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + switch (p->multipart_state) { + case MULTIPART_INACTIVE: + upb_status_seterrmsg( + p->status, "Internal error: unexpected state MULTIPART_INACTIVE"); + return false; + + case MULTIPART_ACCUMULATE: + if (!accumulate_append(p, buf, len, can_alias)) { + return false; + } + break; + + case MULTIPART_PUSHEAGERLY: { + const upb_bufhandle *handle = can_alias ? p->handle : NULL; + upb_sink_putstring(p->top->sink, p->string_selector, buf, len, handle); + break; + } + } + + return true; +} + +/* Note: this invalidates the accumulate buffer! Call only after reading its + * contents. */ +static void multipart_end(upb_json_parser *p) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_INACTIVE; + accumulate_clear(p); +} + + +/* Input capture **************************************************************/ + +/* Functionality for capturing a region of the input as text. Gracefully + * handles the case where a buffer seam occurs in the middle of the captured + * region. */ + +static void capture_begin(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + UPB_ASSERT(p->capture == NULL); + p->capture = ptr; +} + +static bool capture_end(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->capture); + if (multipart_text(p, p->capture, ptr - p->capture, true)) { + p->capture = NULL; + return true; + } else { + return false; + } +} + +/* This is called at the end of each input buffer (ie. when we have hit a + * buffer seam). If we are in the middle of capturing the input, this + * processes the unprocessed capture region. */ +static void capture_suspend(upb_json_parser *p, const char **ptr) { + if (!p->capture) return; + + if (multipart_text(p, p->capture, *ptr - p->capture, false)) { + /* We use this as a signal that we were in the middle of capturing, and + * that capturing should resume at the beginning of the next buffer. + * + * We can't use *ptr here, because we have no guarantee that this pointer + * will be valid when we resume (if the underlying memory is freed, then + * using the pointer at all, even to compare to NULL, is likely undefined + * behavior). */ + p->capture = &suspend_capture; + } else { + /* Need to back up the pointer to the beginning of the capture, since + * we were not able to actually preserve it. */ + *ptr = p->capture; + } +} + +static void capture_resume(upb_json_parser *p, const char *ptr) { + if (p->capture) { + UPB_ASSERT(p->capture == &suspend_capture); + p->capture = ptr; + } +} + + +/* Callbacks from the parser **************************************************/ + +/* These are the functions called directly from the parser itself. + * We define these in the same order as their declarations in the parser. */ + +static char escape_char(char in) { + switch (in) { + case 'r': return '\r'; + case 't': return '\t'; + case 'n': return '\n'; + case 'f': return '\f'; + case 'b': return '\b'; + case '/': return '/'; + case '"': return '"'; + case '\\': return '\\'; + default: + UPB_ASSERT(0); + return 'x'; + } +} + +static bool escape(upb_json_parser *p, const char *ptr) { + char ch = escape_char(*ptr); + return multipart_text(p, &ch, 1, false); +} + +static void start_hex(upb_json_parser *p) { + p->digit = 0; +} + +static void hexdigit(upb_json_parser *p, const char *ptr) { + char ch = *ptr; + + p->digit <<= 4; + + if (ch >= '0' && ch <= '9') { + p->digit += (ch - '0'); + } else if (ch >= 'a' && ch <= 'f') { + p->digit += ((ch - 'a') + 10); + } else { + UPB_ASSERT(ch >= 'A' && ch <= 'F'); + p->digit += ((ch - 'A') + 10); + } +} + +static bool end_hex(upb_json_parser *p) { + uint32_t codepoint = p->digit; + + /* emit the codepoint as UTF-8. */ + char utf8[3]; /* support \u0000 -- \uFFFF -- need only three bytes. */ + int length = 0; + if (codepoint <= 0x7F) { + utf8[0] = codepoint; + length = 1; + } else if (codepoint <= 0x07FF) { + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x1F) | 0xC0; + length = 2; + } else /* codepoint <= 0xFFFF */ { + utf8[2] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x0F) | 0xE0; + length = 3; + } + /* TODO(haberman): Handle high surrogates: if codepoint is a high surrogate + * we have to wait for the next escape to get the full code point). */ + + return multipart_text(p, utf8, length, false); +} + +static void start_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_number(upb_json_parser *p, const char *ptr) { + if (is_top_level(p)) { + if (is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NUMBERVALUE); + } else { + return false; + } + } else if (does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NUMBERVALUE); + } + + multipart_startaccum(p); + capture_begin(p, ptr); + return true; +} + +static bool parse_number(upb_json_parser *p, bool is_quoted); + +static bool end_number_nontop(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + return parse_number(p, false); +} + +static bool end_number(upb_json_parser *p, const char *ptr) { + if (!end_number_nontop(p, ptr)) { + return false; + } + + if (does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +/* |buf| is NULL-terminated. |buf| itself will never include quotes; + * |is_quoted| tells us whether this text originally appeared inside quotes. */ +static bool parse_number_from_buffer(upb_json_parser *p, const char *buf, + bool is_quoted) { + size_t len = strlen(buf); + const char *bufend = buf + len; + char *end; + upb_fieldtype_t type = upb_fielddef_type(p->top->f); + double val; + double dummy; + double inf = UPB_INFINITY; + + errno = 0; + + if (len == 0 || buf[0] == ' ') { + return false; + } + + /* For integer types, first try parsing with integer-specific routines. + * If these succeed, they will be more accurate for int64/uint64 than + * strtod(). + */ + switch (type) { + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else if (val > INT32_MAX || val < INT32_MIN) { + return false; + } else { + upb_sink_putint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT32: { + unsigned long val = strtoul(buf, &end, 0); + if (end != bufend) { + break; + } else if (val > UINT32_MAX || errno == ERANGE) { + return false; + } else { + upb_sink_putuint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + /* XXX: We can't handle [u]int64 properly on 32-bit machines because + * strto[u]ll isn't in C89. */ + case UPB_TYPE_INT64: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else { + upb_sink_putint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT64: { + unsigned long val = strtoul(p->accumulated, &end, 0); + if (end != bufend) { + break; + } else if (errno == ERANGE) { + return false; + } else { + upb_sink_putuint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + default: + break; + } + + if (type != UPB_TYPE_DOUBLE && type != UPB_TYPE_FLOAT && is_quoted) { + /* Quoted numbers for integer types are not allowed to be in double form. */ + return false; + } + + if (len == strlen("Infinity") && strcmp(buf, "Infinity") == 0) { + /* C89 does not have an INFINITY macro. */ + val = inf; + } else if (len == strlen("-Infinity") && strcmp(buf, "-Infinity") == 0) { + val = -inf; + } else { + val = strtod(buf, &end); + if (errno == ERANGE || end != bufend) { + return false; + } + } + + switch (type) { +#define CASE(capitaltype, smalltype, ctype, min, max) \ + case UPB_TYPE_ ## capitaltype: { \ + if (modf(val, &dummy) != 0 || val > max || val < min) { \ + return false; \ + } else { \ + upb_sink_put ## smalltype(p->top->sink, parser_getsel(p), \ + (ctype)val); \ + return true; \ + } \ + break; \ + } + case UPB_TYPE_ENUM: + CASE(INT32, int32, int32_t, INT32_MIN, INT32_MAX); + CASE(INT64, int64, int64_t, INT64_MIN, INT64_MAX); + CASE(UINT32, uint32, uint32_t, 0, UINT32_MAX); + CASE(UINT64, uint64, uint64_t, 0, UINT64_MAX); +#undef CASE + + case UPB_TYPE_DOUBLE: + upb_sink_putdouble(p->top->sink, parser_getsel(p), val); + return true; + case UPB_TYPE_FLOAT: + if ((val > FLT_MAX || val < -FLT_MAX) && val != inf && val != -inf) { + return false; + } else { + upb_sink_putfloat(p->top->sink, parser_getsel(p), val); + return true; + } + default: + return false; + } +} + +static bool parse_number(upb_json_parser *p, bool is_quoted) { + size_t len; + const char *buf; + + /* strtol() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + if (!multipart_text(p, "\0", 1, false)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (parse_number_from_buffer(p, buf, is_quoted)) { + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "error parsing number: %s", buf); + multipart_end(p); + return false; + } +} + +static bool parser_putbool(upb_json_parser *p, bool val) { + bool ok; + + if (p->top->f == NULL) { + return true; + } + + if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { + upb_status_seterrf(p->status, + "Boolean value specified for non-bool field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + ok = upb_sink_putbool(p->top->sink, parser_getsel(p), val); + UPB_ASSERT(ok); + + return true; +} + +static bool end_bool(upb_json_parser *p, bool val) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_BOOLVALUE); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_BOOLVALUE)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_BOOLVALUE); + } + + if (p->top->is_unknown_field) { + return true; + } + + if (!parser_putbool(p, val)) { + return false; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static bool end_null(upb_json_parser *p) { + const char *zero_ptr = "0"; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + + /* Fill null_value field. */ + multipart_startaccum(p); + capture_begin(p, zero_ptr); + capture_end(p, zero_ptr + 1); + parse_number(p, false); + + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + + return true; +} + +static bool start_any_stringval(upb_json_parser *p) { + multipart_startaccum(p); + return true; +} + +static bool start_stringval(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_string_wrapper_object(p) || + is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + start_fieldmask_object(p); + return true; + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + start_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRINGVALUE); + } else { + return false; + } + } else if (does_string_wrapper_start(p) || + does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (does_fieldmask_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_fieldmask_object(p); + return true; + } else if (is_wellknown_field(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_field(p, UPB_WELLKNOWN_DURATION)) { + if (!start_subobject(p)) { + return false; + } + start_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_STRINGVALUE); + } + + if (p->top->f == NULL) { + multipart_startaccum(p); + return true; + } + + if (p->top->is_any) { + return start_any_stringval(p); + } + + if (upb_fielddef_isstring(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + if (upb_fielddef_type(p->top->f) == UPB_TYPE_STRING) { + /* For STRING fields we push data directly to the handlers as it is + * parsed. We don't do this yet for BYTES fields, because our base64 + * decoder is not streaming. + * + * TODO(haberman): make base64 decoding streaming also. */ + multipart_start(p, getsel_for_handlertype(p, UPB_HANDLER_STRING)); + return true; + } else { + multipart_startaccum(p); + return true; + } + } else if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL && + upb_fielddef_type(p->top->f) != UPB_TYPE_MESSAGE) { + /* No need to push a frame -- numeric values in quotes remain in the + * current parser frame. These values must accmulate so we can convert + * them all at once at the end. */ + multipart_startaccum(p); + return true; + } else { + upb_status_seterrf(p->status, + "String specified for bool or submessage field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool end_any_stringval(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Set type_url */ + upb_selector_t sel; + upb_jsonparser_frame *inner; + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + multipart_end(p); + + /* Resolve type url */ + if (strncmp(buf, "type.googleapis.com/", 20) == 0 && len > 20) { + const upb_msgdef *payload_type = NULL; + buf += 20; + len -= 20; + + payload_type = upb_symtab_lookupmsg2(p->symtab, buf, len); + if (payload_type == NULL) { + upb_status_seterrf( + p->status, "Cannot find packed type: %.*s\n", (int)len, buf); + return false; + } + + json_parser_any_frame_set_payload_type(p, p->top->any_frame, payload_type); + + return true; + } else { + upb_status_seterrf( + p->status, "Invalid type url: %.*s\n", (int)len, buf); + return false; + } +} + +static bool end_stringval_nontop(upb_json_parser *p) { + bool ok = true; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + multipart_end(p); + return true; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_stringval(p); + } + + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_BYTES: + if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + /* Fall through. */ + + case UPB_TYPE_STRING: { + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + break; + } + + case UPB_TYPE_ENUM: { + /* Resolve enum symbolic name to integer value. */ + const upb_enumdef *enumdef = upb_fielddef_enumsubdef(p->top->f); + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + int32_t int_val = 0; + ok = upb_enumdef_ntoi(enumdef, buf, len, &int_val); + + if (ok) { + upb_selector_t sel = parser_getsel(p); + upb_sink_putint32(p->top->sink, sel, int_val); + } else { + upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf); + } + + break; + } + + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + ok = parse_number(p, true); + break; + + default: + UPB_ASSERT(false); + upb_status_seterrmsg(p->status, "Internal error in JSON decoder"); + ok = false; + break; + } + + multipart_end(p); + + return ok; +} + +static bool end_stringval(upb_json_parser *p) { + /* FieldMask's stringvals have been ended when handling them. Only need to + * close FieldMask here.*/ + if (does_fieldmask_end(p)) { + end_fieldmask_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (!end_stringval_nontop(p)) { + return false; + } + + if (does_string_wrapper_end(p) || + does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION) || + is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + end_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static void start_duration_base(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_duration_base(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char seconds_buf[14]; + char nanos_buf[12]; + char *end; + int64_t seconds = 0; + int32_t nanos = 0; + double val = 0.0; + const char *seconds_membername = "seconds"; + const char *nanos_membername = "nanos"; + size_t fraction_start; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + memset(seconds_buf, 0, 14); + memset(nanos_buf, 0, 12); + + /* Find out base end. The maximus duration is 315576000000, which cannot be + * represented by double without losing precision. Thus, we need to handle + * fraction and base separately. */ + for (fraction_start = 0; fraction_start < len && buf[fraction_start] != '.'; + fraction_start++); + + /* Parse base */ + memcpy(seconds_buf, buf, fraction_start); + seconds = strtol(seconds_buf, &end, 10); + if (errno == ERANGE || end != seconds_buf + fraction_start) { + upb_status_seterrf(p->status, "error parsing duration: %s", + seconds_buf); + return false; + } + + if (seconds > 315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "maximum acceptable value is " + "315576000000"); + return false; + } + + if (seconds < -315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "minimum acceptable value is " + "-315576000000"); + return false; + } + + /* Parse fraction */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf + fraction_start, len - fraction_start); + val = strtod(nanos_buf, &end); + if (errno == ERANGE || end != nanos_buf + len - fraction_start + 1) { + upb_status_seterrf(p->status, "error parsing duration: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + if (seconds < 0) nanos = -nanos; + + /* Clean up buffer */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous arena */ + multipart_startaccum(p); + + return true; +} + +static int parse_timestamp_number(upb_json_parser *p) { + size_t len; + const char *buf; + int val; + + /* atoi() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + multipart_text(p, "\0", 1, false); + + buf = accumulate_getptr(p, &len); + val = atoi(buf); + multipart_end(p); + multipart_startaccum(p); + + return val; +} + +static void start_year(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_year(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_year = parse_timestamp_number(p) - 1900; + return true; +} + +static void start_month(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_month(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mon = parse_timestamp_number(p) - 1; + return true; +} + +static void start_day(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_day(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mday = parse_timestamp_number(p); + return true; +} + +static void start_hour(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_hour(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_hour = parse_timestamp_number(p); + return true; +} + +static void start_minute(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_minute(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_min = parse_timestamp_number(p); + return true; +} + +static void start_second(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_second(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_sec = parse_timestamp_number(p); + return true; +} + +static void start_timestamp_base(upb_json_parser *p) { + memset(&p->tm, 0, sizeof(struct tm)); +} + +static void start_timestamp_fraction(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_timestamp_fraction(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char nanos_buf[12]; + char *end; + double val = 0.0; + int32_t nanos; + const char *nanos_membername = "nanos"; + + memset(nanos_buf, 0, 12); + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (len > 10) { + upb_status_seterrf(p->status, + "error parsing timestamp: at most 9-digit fraction."); + return false; + } + + /* Parse nanos */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf, len); + val = strtod(nanos_buf, &end); + + if (errno == ERANGE || end != nanos_buf + len + 1) { + upb_status_seterrf(p->status, "error parsing timestamp nanos: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + + /* Clean up previous environment */ + multipart_end(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_timestamp_zone(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +#define EPOCH_YEAR 1970 +#define TM_YEAR_BASE 1900 + +static bool isleap(int year) { + return (year % 4) == 0 && (year % 100 != 0 || (year % 400) == 0); +} + +const unsigned short int __mon_yday[2][13] = { + /* Normal years. */ + { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, + /* Leap years. */ + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } +}; + +int64_t epoch(int year, int yday, int hour, int min, int sec) { + int64_t years = year - EPOCH_YEAR; + + int64_t leap_days = years / 4 - years / 100 + years / 400; + + int64_t days = years * 365 + yday + leap_days; + int64_t hours = days * 24 + hour; + int64_t mins = hours * 60 + min; + int64_t secs = mins * 60 + sec; + return secs; +} + + +static int64_t upb_mktime(const struct tm *tp) { + int sec = tp->tm_sec; + int min = tp->tm_min; + int hour = tp->tm_hour; + int mday = tp->tm_mday; + int mon = tp->tm_mon; + int year = tp->tm_year + TM_YEAR_BASE; + + /* Calculate day of year from year, month, and day of month. */ + int mon_yday = ((__mon_yday[isleap(year)][mon]) - 1); + int yday = mon_yday + mday; + + return epoch(year, yday, hour, min, sec); +} + +static bool end_timestamp_zone(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + int hours; + int64_t seconds; + const char *seconds_membername = "seconds"; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (buf[0] != 'Z') { + if (sscanf(buf + 1, "%2d:00", &hours) != 1) { + upb_status_seterrf(p->status, "error parsing timestamp offset"); + return false; + } + + if (buf[0] == '+') { + hours = -hours; + } + + p->tm.tm_hour += hours; + } + + /* Normalize tm */ + seconds = upb_mktime(&p->tm); + + /* Check timestamp boundary */ + if (seconds < -62135596800) { + upb_status_seterrf(p->status, "error parsing timestamp: " + "minimum acceptable value is " + "0001-01-01T00:00:00Z"); + return false; + } + + /* Clean up previous environment */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_fieldmask_path(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + multipart_startaccum(p); + return true; +} + +static bool lower_camel_push( + upb_json_parser *p, upb_selector_t sel, const char *ptr, size_t len) { + const char *limit = ptr + len; + bool first = true; + for (;ptr < limit; ptr++) { + if (*ptr >= 'A' && *ptr <= 'Z' && !first) { + char lower = tolower(*ptr); + upb_sink_putstring(p->top->sink, sel, "_", 1, NULL); + upb_sink_putstring(p->top->sink, sel, &lower, 1, NULL); + } else { + upb_sink_putstring(p->top->sink, sel, ptr, 1, NULL); + } + first = false; + } + return true; +} + +static bool end_fieldmask_path(upb_json_parser *p) { + upb_selector_t sel; + + if (!lower_camel_push( + p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + + multipart_end(p); + return true; +} + +static void start_member(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + multipart_startaccum(p); +} + +/* Helper: invoked during parse_mapentry() to emit the mapentry message's key + * field based on the current contents of the accumulate buffer. */ +static bool parse_mapentry_key(upb_json_parser *p) { + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Emit the key field. We do a bit of ad-hoc parsing here because the + * parser state machine has already decided that this is a string field + * name, and we are reinterpreting it as some arbitrary key type. In + * particular, integer and bool keys are quoted, so we need to parse the + * quoted string contents here. */ + + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_KEY); + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no key"); + return false; + } + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + /* Invoke end_number. The accum buffer has the number's text already. */ + if (!parse_number(p, true)) { + return false; + } + break; + case UPB_TYPE_BOOL: + if (len == 4 && !strncmp(buf, "true", 4)) { + if (!parser_putbool(p, true)) { + return false; + } + } else if (len == 5 && !strncmp(buf, "false", 5)) { + if (!parser_putbool(p, false)) { + return false; + } + } else { + upb_status_seterrmsg(p->status, + "Map bool key not 'true' or 'false'"); + return false; + } + multipart_end(p); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + upb_sink subsink; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, len, &subsink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(subsink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(subsink, sel); + multipart_end(p); + break; + } + default: + upb_status_seterrmsg(p->status, "Invalid field type for map key"); + return false; + } + + return true; +} + +/* Helper: emit one map entry (as a submessage in the map field sequence). This + * is invoked from end_membername(), at the end of the map entry's key string, + * with the map key in the accumulate buffer. It parses the key from that + * buffer, emits the handler calls to start the mapentry submessage (setting up + * its subframe in the process), and sets up state in the subframe so that the + * value parser (invoked next) will emit the mapentry's value field and then + * end the mapentry message. */ + +static bool handle_mapentry(upb_json_parser *p) { + const upb_fielddef *mapfield; + const upb_msgdef *mapentrymsg; + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Map entry: p->top->sink is the seq frame, so we need to start a frame + * for the mapentry itself, and then set |f| in that frame so that the map + * value field is parsed, and also set a flag to end the frame after the + * map-entry value is parsed. */ + if (!check_stack(p)) return false; + + mapfield = p->top->mapfield; + mapentrymsg = upb_fielddef_msgsubdef(mapfield); + + inner = start_jsonparser_frame(p); + p->top->f = mapfield; + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = mapentrymsg; + inner->mapfield = mapfield; + + /* Don't set this to true *yet* -- we reuse parsing handlers below to push + * the key field value to the sink, and these handlers will pop the frame + * if they see is_mapentry (when invoked by the parser state machine, they + * would have just seen the map-entry value, not key). */ + inner->is_mapentry = false; + p->top = inner; + + /* send STARTMSG in submsg frame. */ + upb_sink_startmsg(p->top->sink); + + parse_mapentry_key(p); + + /* Set up the value field to receive the map-entry value. */ + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_VALUE); + p->top->is_mapentry = true; /* set up to pop frame after value is parsed. */ + p->top->mapfield = mapfield; + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no value"); + return false; + } + + return true; +} + +static bool end_membername(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + + if (!p->top->m) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_membername(p); + } else if (p->top->is_map) { + return handle_mapentry(p); + } else { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (upb_strtable_lookup2(p->top->name_table, buf, len, &v)) { + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + + return true; + } else if (p->ignore_json_unknown) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + return false; + } + } +} + +static bool end_any_membername(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (len == 5 && strncmp(buf, "@type", len) == 0) { + upb_strtable_lookup2(p->top->name_table, "type_url", 8, &v); + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + return true; + } else { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } +} + +static void end_member(upb_json_parser *p) { + /* If we just parsed a map-entry value, end that frame too. */ + if (p->top->is_mapentry) { + upb_selector_t sel; + bool ok; + const upb_fielddef *mapfield; + + UPB_ASSERT(p->top > p->stack); + /* send ENDMSG on submsg. */ + upb_sink_endmsg(p->top->sink, p->status); + mapfield = p->top->mapfield; + + /* send ENDSUBMSG in repeated-field-of-mapentries frame. */ + p->top--; + ok = upb_handlers_getselector(mapfield, UPB_HANDLER_ENDSUBMSG, &sel); + UPB_ASSERT(ok); + upb_sink_endsubmsg(p->top->sink, sel); + } + + p->top->f = NULL; + p->top->is_unknown_field = false; +} + +static void start_any_member(upb_json_parser *p, const char *ptr) { + start_member(p); + json_parser_any_frame_set_after_type_url_start_once(p->top->any_frame, ptr); +} + +static void end_any_member(upb_json_parser *p, const char *ptr) { + json_parser_any_frame_set_before_type_url_end(p->top->any_frame, ptr); + end_member(p); +} + +static bool start_subobject(upb_json_parser *p) { + if (p->top->is_unknown_field) { + if (!check_stack(p)) return false; + + p->top = start_jsonparser_frame(p); + return true; + } + + if (upb_fielddef_ismap(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a map. Start a new parser frame in a repeated-field + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->mapfield = p->top->f; + inner->is_map = true; + p->top = inner; + + return true; + } else if (upb_fielddef_issubmsg(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a subobject. Start a new parser frame in the submsg + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + set_name_table(p, inner); + p->top = inner; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + + return true; + } else { + upb_status_seterrf(p->status, + "Object specified for non-message/group field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool start_subobject_full(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + start_structvalue_object(p); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_STRUCT)) { + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } + + return start_subobject(p); +} + +static void end_subobject(upb_json_parser *p) { + if (is_top_level(p)) { + return; + } + + if (p->top->is_map) { + upb_selector_t sel; + p->top--; + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + } else { + upb_selector_t sel; + bool is_unknown = p->top->m == NULL; + p->top--; + if (!is_unknown) { + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(p->top->sink, sel); + } + } +} + +static void end_subobject_full(upb_json_parser *p) { + end_subobject(p); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + end_structvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static bool start_array(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + start_listvalue_object(p); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_LISTVALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } + + if (p->top->is_unknown_field) { + inner = start_jsonparser_frame(p); + inner->is_unknown_field = true; + p->top = inner; + + return true; + } + + if (!upb_fielddef_isseq(p->top->f)) { + upb_status_seterrf(p->status, + "Array specified for non-repeated field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + inner->is_repeated = true; + p->top = inner; + + return true; +} + +static void end_array(upb_json_parser *p) { + upb_selector_t sel; + + UPB_ASSERT(p->top > p->stack); + + p->top--; + + if (p->top->is_unknown_field) { + return; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + end_listvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static void start_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_startmsg(p->top->sink); + } +} + +static void end_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_endmsg(p->top->sink, p->status); + } +} + +static void start_any_object(upb_json_parser *p, const char *ptr) { + start_object(p); + p->top->any_frame->before_type_url_start = ptr; + p->top->any_frame->before_type_url_end = ptr; +} + +static bool end_any_object(upb_json_parser *p, const char *ptr) { + const char *value_membername = "value"; + bool is_well_known_packed = false; + const char *packed_end = ptr + 1; + upb_selector_t sel; + upb_jsonparser_frame *inner; + + if (json_parser_any_frame_has_value(p->top->any_frame) && + !json_parser_any_frame_has_type_url(p->top->any_frame)) { + upb_status_seterrmsg(p->status, "No valid type url"); + return false; + } + + /* Well known types data is represented as value field. */ + if (upb_msgdef_wellknowntype(p->top->any_frame->parser->top->m) != + UPB_WELLKNOWN_UNSPECIFIED) { + is_well_known_packed = true; + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + p->top->any_frame->before_type_url_start = + memchr(p->top->any_frame->before_type_url_start, ':', + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start); + if (p->top->any_frame->before_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "invalid data for well known type."); + return false; + } + p->top->any_frame->before_type_url_start++; + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + p->top->any_frame->after_type_url_start = + memchr(p->top->any_frame->after_type_url_start, ':', + (ptr + 1) - + p->top->any_frame->after_type_url_start); + if (p->top->any_frame->after_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "Invalid data for well known type."); + return false; + } + p->top->any_frame->after_type_url_start++; + packed_end = ptr; + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->before_type_url_start, + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "{", 1, NULL)) { + return false; + } + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame) && + json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, ",", 1, NULL)) { + return false; + } + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->after_type_url_start, + packed_end - p->top->any_frame->after_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "}", 1, NULL)) { + return false; + } + } + } + + if (!end(p->top->any_frame->parser, NULL)) { + return false; + } + + p->top->is_any = false; + + /* Set value */ + start_member(p); + capture_begin(p, value_membername); + capture_end(p, value_membername + 5); + end_membername(p); + + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, p->top->any_frame->stringsink.ptr, + p->top->any_frame->stringsink.len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + end_member(p); + + end_object(p); + + /* Deallocate any parse frame. */ + json_parser_any_frame_free(p->top->any_frame); + + return true; +} + +static bool is_string_wrapper(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_STRINGVALUE || + type == UPB_WELLKNOWN_BYTESVALUE; +} + +static bool is_fieldmask(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_FIELDMASK; +} + +static void start_fieldmask_object(upb_json_parser *p) { + const char *membername = "paths"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); + + start_array(p); +} + +static void end_fieldmask_object(upb_json_parser *p) { + end_array(p); + end_member(p); + end_object(p); +} + +static void start_wrapper_object(upb_json_parser *p) { + const char *membername = "value"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); +} + +static void end_wrapper_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_value_object(upb_json_parser *p, int value_type) { + const char *nullmember = "null_value"; + const char *numbermember = "number_value"; + const char *stringmember = "string_value"; + const char *boolmember = "bool_value"; + const char *structmember = "struct_value"; + const char *listmember = "list_value"; + const char *membername = ""; + + switch (value_type) { + case VALUE_NULLVALUE: + membername = nullmember; + break; + case VALUE_NUMBERVALUE: + membername = numbermember; + break; + case VALUE_STRINGVALUE: + membername = stringmember; + break; + case VALUE_BOOLVALUE: + membername = boolmember; + break; + case VALUE_STRUCTVALUE: + membername = structmember; + break; + case VALUE_LISTVALUE: + membername = listmember; + break; + } + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_value_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_listvalue_object(upb_json_parser *p) { + const char *membername = "values"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_listvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_structvalue_object(upb_json_parser *p) { + const char *membername = "fields"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_structvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static bool is_top_level(upb_json_parser *p) { + return p->top == p->stack && p->top->f == NULL && !p->top->is_unknown_field; +} + +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->m != NULL && upb_msgdef_wellknowntype(p->top->m) == type; +} + +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + (upb_msgdef_wellknowntype(upb_fielddef_msgsubdef(p->top->f)) + == type); +} + +static bool does_number_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + upb_msgdef_isnumberwrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_number_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool is_number_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool does_string_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_string_wrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_string_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool is_string_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool does_fieldmask_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_fieldmask(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_fieldmask_end(upb_json_parser *p) { + return p->top->m != NULL && is_fieldmask(p->top->m); +} + +#define CHECK_RETURN_TOP(x) if (!(x)) goto error + + +/* The actual parser **********************************************************/ + +/* What follows is the Ragel parser itself. The language is specified in Ragel + * and the actions call our C functions above. + * + * Ragel has an extensive set of functionality, and we use only a small part of + * it. There are many action types but we only use a few: + * + * ">" -- transition into a machine + * "%" -- transition out of a machine + * "@" -- transition into a final state of a machine. + * + * "@" transitions are tricky because a machine can transition into a final + * state repeatedly. But in some cases we know this can't happen, for example + * a string which is delimited by a final '"' can only transition into its + * final state once, when the closing '"' is seen. */ + + +#line 2794 "upb/json/parser.rl" + + + +#line 2597 "upb/json/parser.c" +static const char _json_actions[] = { + 0, 1, 0, 1, 1, 1, 3, 1, + 4, 1, 6, 1, 7, 1, 8, 1, + 9, 1, 11, 1, 12, 1, 13, 1, + 14, 1, 15, 1, 16, 1, 17, 1, + 18, 1, 19, 1, 20, 1, 22, 1, + 23, 1, 24, 1, 35, 1, 37, 1, + 39, 1, 40, 1, 42, 1, 43, 1, + 44, 1, 46, 1, 48, 1, 49, 1, + 50, 1, 51, 1, 53, 1, 54, 2, + 4, 9, 2, 5, 6, 2, 7, 3, + 2, 7, 9, 2, 21, 26, 2, 25, + 10, 2, 27, 28, 2, 29, 30, 2, + 32, 34, 2, 33, 31, 2, 38, 36, + 2, 40, 42, 2, 45, 2, 2, 46, + 54, 2, 47, 36, 2, 49, 54, 2, + 50, 54, 2, 51, 54, 2, 52, 41, + 2, 53, 54, 3, 32, 34, 35, 4, + 21, 26, 27, 28 +}; + +static const short _json_key_offsets[] = { + 0, 0, 12, 13, 18, 23, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, + 38, 43, 44, 48, 53, 58, 63, 67, + 71, 74, 77, 79, 83, 87, 89, 91, + 96, 98, 100, 109, 115, 121, 127, 133, + 135, 139, 142, 144, 146, 149, 150, 154, + 156, 158, 160, 162, 163, 165, 167, 168, + 170, 172, 173, 175, 177, 178, 180, 182, + 183, 185, 187, 191, 193, 195, 196, 197, + 198, 199, 201, 206, 208, 210, 212, 221, + 222, 222, 222, 227, 232, 237, 238, 239, + 240, 241, 241, 242, 243, 244, 244, 245, + 246, 247, 247, 252, 253, 257, 262, 267, + 272, 276, 276, 279, 282, 285, 288, 291, + 294, 294, 294, 294, 294, 294 +}; + +static const char _json_trans_keys[] = { + 32, 34, 45, 91, 102, 110, 116, 123, + 9, 13, 48, 57, 34, 32, 93, 125, + 9, 13, 32, 44, 93, 9, 13, 32, + 93, 125, 9, 13, 97, 108, 115, 101, + 117, 108, 108, 114, 117, 101, 32, 34, + 125, 9, 13, 34, 32, 58, 9, 13, + 32, 93, 125, 9, 13, 32, 44, 125, + 9, 13, 32, 44, 125, 9, 13, 32, + 34, 9, 13, 45, 48, 49, 57, 48, + 49, 57, 46, 69, 101, 48, 57, 69, + 101, 48, 57, 43, 45, 48, 57, 48, + 57, 48, 57, 46, 69, 101, 48, 57, + 34, 92, 34, 92, 34, 47, 92, 98, + 102, 110, 114, 116, 117, 48, 57, 65, + 70, 97, 102, 48, 57, 65, 70, 97, + 102, 48, 57, 65, 70, 97, 102, 48, + 57, 65, 70, 97, 102, 34, 92, 45, + 48, 49, 57, 48, 49, 57, 46, 115, + 48, 57, 115, 48, 57, 34, 46, 115, + 48, 57, 48, 57, 48, 57, 48, 57, + 48, 57, 45, 48, 57, 48, 57, 45, + 48, 57, 48, 57, 84, 48, 57, 48, + 57, 58, 48, 57, 48, 57, 58, 48, + 57, 48, 57, 43, 45, 46, 90, 48, + 57, 48, 57, 58, 48, 48, 34, 48, + 57, 43, 45, 90, 48, 57, 34, 44, + 34, 44, 34, 44, 34, 45, 91, 102, + 110, 116, 123, 48, 57, 34, 32, 93, + 125, 9, 13, 32, 44, 93, 9, 13, + 32, 93, 125, 9, 13, 97, 108, 115, + 101, 117, 108, 108, 114, 117, 101, 32, + 34, 125, 9, 13, 34, 32, 58, 9, + 13, 32, 93, 125, 9, 13, 32, 44, + 125, 9, 13, 32, 44, 125, 9, 13, + 32, 34, 9, 13, 32, 9, 13, 32, + 9, 13, 32, 9, 13, 32, 9, 13, + 32, 9, 13, 32, 9, 13, 0 +}; + +static const char _json_single_lengths[] = { + 0, 8, 1, 3, 3, 3, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 3, 1, 2, 3, 3, 3, 2, 2, + 1, 3, 0, 2, 2, 0, 0, 3, + 2, 2, 9, 0, 0, 0, 0, 2, + 2, 1, 2, 0, 1, 1, 2, 0, + 0, 0, 0, 1, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 1, + 0, 0, 4, 0, 0, 1, 1, 1, + 1, 0, 3, 2, 2, 2, 7, 1, + 0, 0, 3, 3, 3, 1, 1, 1, + 1, 0, 1, 1, 1, 0, 1, 1, + 1, 0, 3, 1, 2, 3, 3, 3, + 2, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0 +}; + +static const char _json_range_lengths[] = { + 0, 2, 0, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 3, 0, + 1, 1, 0, 1, 1, 0, 1, 1, + 1, 1, 1, 0, 1, 1, 0, 1, + 1, 0, 1, 1, 0, 1, 1, 0, + 1, 1, 0, 1, 1, 0, 0, 0, + 0, 1, 1, 0, 0, 0, 1, 0, + 0, 0, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 1, 1, 1, + 1, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0 +}; + +static const short _json_index_offsets[] = { + 0, 0, 11, 13, 18, 23, 28, 30, + 32, 34, 36, 38, 40, 42, 44, 46, + 48, 53, 55, 59, 64, 69, 74, 78, + 82, 85, 89, 91, 95, 99, 101, 103, + 108, 111, 114, 124, 128, 132, 136, 140, + 143, 147, 150, 153, 155, 158, 160, 164, + 166, 168, 170, 172, 174, 176, 178, 180, + 182, 184, 186, 188, 190, 192, 194, 196, + 198, 200, 202, 207, 209, 211, 213, 215, + 217, 219, 221, 226, 229, 232, 235, 244, + 246, 247, 248, 253, 258, 263, 265, 267, + 269, 271, 272, 274, 276, 278, 279, 281, + 283, 285, 286, 291, 293, 297, 302, 307, + 312, 316, 317, 320, 323, 326, 329, 332, + 335, 336, 337, 338, 339, 340 +}; + +static const unsigned char _json_indicies[] = { + 0, 2, 3, 4, 5, 6, 7, 8, + 0, 3, 1, 9, 1, 11, 12, 1, + 11, 10, 13, 14, 12, 13, 1, 14, + 1, 1, 14, 10, 15, 1, 16, 1, + 17, 1, 18, 1, 19, 1, 20, 1, + 21, 1, 22, 1, 23, 1, 24, 1, + 25, 26, 27, 25, 1, 28, 1, 29, + 30, 29, 1, 30, 1, 1, 30, 31, + 32, 33, 34, 32, 1, 35, 36, 27, + 35, 1, 36, 26, 36, 1, 37, 38, + 39, 1, 38, 39, 1, 41, 42, 42, + 40, 43, 1, 42, 42, 43, 40, 44, + 44, 45, 1, 45, 1, 45, 40, 41, + 42, 42, 39, 40, 47, 48, 46, 50, + 51, 49, 52, 52, 52, 52, 52, 52, + 52, 52, 53, 1, 54, 54, 54, 1, + 55, 55, 55, 1, 56, 56, 56, 1, + 57, 57, 57, 1, 59, 60, 58, 61, + 62, 63, 1, 64, 65, 1, 66, 67, + 1, 68, 1, 67, 68, 1, 69, 1, + 66, 67, 65, 1, 70, 1, 71, 1, + 72, 1, 73, 1, 74, 1, 75, 1, + 76, 1, 77, 1, 78, 1, 79, 1, + 80, 1, 81, 1, 82, 1, 83, 1, + 84, 1, 85, 1, 86, 1, 87, 1, + 88, 1, 89, 89, 90, 91, 1, 92, + 1, 93, 1, 94, 1, 95, 1, 96, + 1, 97, 1, 98, 1, 99, 99, 100, + 98, 1, 102, 1, 101, 104, 105, 103, + 1, 1, 101, 106, 107, 108, 109, 110, + 111, 112, 107, 1, 113, 1, 114, 115, + 117, 118, 1, 117, 116, 119, 120, 118, + 119, 1, 120, 1, 1, 120, 116, 121, + 1, 122, 1, 123, 1, 124, 1, 125, + 126, 1, 127, 1, 128, 1, 129, 130, + 1, 131, 1, 132, 1, 133, 134, 135, + 136, 134, 1, 137, 1, 138, 139, 138, + 1, 139, 1, 1, 139, 140, 141, 142, + 143, 141, 1, 144, 145, 136, 144, 1, + 145, 135, 145, 1, 146, 147, 147, 1, + 148, 148, 1, 149, 149, 1, 150, 150, + 1, 151, 151, 1, 152, 152, 1, 1, + 1, 1, 1, 1, 1, 0 +}; + +static const char _json_trans_targs[] = { + 1, 0, 2, 107, 3, 6, 10, 13, + 16, 106, 4, 3, 106, 4, 5, 7, + 8, 9, 108, 11, 12, 109, 14, 15, + 110, 16, 17, 111, 18, 18, 19, 20, + 21, 22, 111, 21, 22, 24, 25, 31, + 112, 26, 28, 27, 29, 30, 33, 113, + 34, 33, 113, 34, 32, 35, 36, 37, + 38, 39, 33, 113, 34, 41, 42, 46, + 42, 46, 43, 45, 44, 114, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 73, 72, 68, 69, 70, 71, + 72, 115, 74, 67, 72, 76, 116, 76, + 116, 77, 79, 81, 82, 85, 90, 94, + 98, 80, 117, 117, 83, 82, 80, 83, + 84, 86, 87, 88, 89, 117, 91, 92, + 93, 117, 95, 96, 97, 117, 98, 99, + 105, 100, 100, 101, 102, 103, 104, 105, + 103, 104, 117, 106, 106, 106, 106, 106, + 106 +}; + +static const unsigned char _json_trans_actions[] = { + 0, 0, 113, 107, 53, 0, 0, 0, + 125, 59, 45, 0, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 101, 51, 47, 0, 0, 45, + 49, 49, 104, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 5, 15, + 0, 0, 71, 7, 13, 0, 74, 9, + 9, 9, 77, 80, 11, 37, 37, 37, + 0, 0, 0, 39, 0, 41, 86, 0, + 0, 0, 17, 19, 0, 21, 23, 0, + 25, 27, 0, 29, 31, 0, 33, 35, + 0, 135, 83, 135, 0, 0, 0, 0, + 0, 92, 0, 89, 89, 98, 43, 0, + 131, 95, 113, 107, 53, 0, 0, 0, + 125, 59, 69, 110, 45, 0, 55, 0, + 0, 0, 0, 0, 0, 119, 0, 0, + 0, 122, 0, 0, 0, 116, 0, 101, + 51, 47, 0, 0, 45, 49, 49, 104, + 0, 0, 128, 0, 57, 63, 65, 61, + 67 +}; + +static const unsigned char _json_eof_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 0, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 57, 63, 65, 61, 67, + 0, 0, 0, 0, 0, 0 +}; + +static const int json_start = 1; + +static const int json_en_number_machine = 23; +static const int json_en_string_machine = 32; +static const int json_en_duration_machine = 40; +static const int json_en_timestamp_machine = 47; +static const int json_en_fieldmask_machine = 75; +static const int json_en_value_machine = 78; +static const int json_en_main = 1; + + +#line 2797 "upb/json/parser.rl" + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle) { + upb_json_parser *parser = closure; + + /* Variables used by Ragel's generated code. */ + int cs = parser->current_state; + int *stack = parser->parser_stack; + int top = parser->parser_top; + + const char *p = buf; + const char *pe = buf + size; + const char *eof = &eof_ch; + + parser->handle = handle; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + capture_resume(parser, buf); + + +#line 2875 "upb/json/parser.c" + { + int _klen; + unsigned int _trans; + const char *_acts; + unsigned int _nacts; + const char *_keys; + + if ( p == pe ) + goto _test_eof; + if ( cs == 0 ) + goto _out; +_resume: + _keys = _json_trans_keys + _json_key_offsets[cs]; + _trans = _json_index_offsets[cs]; + + _klen = _json_single_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + _klen - 1; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + ((_upper-_lower) >> 1); + if ( (*p) < *_mid ) + _upper = _mid - 1; + else if ( (*p) > *_mid ) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); + goto _match; + } + } + _keys += _klen; + _trans += _klen; + } + + _klen = _json_range_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( (*p) < _mid[0] ) + _upper = _mid - 2; + else if ( (*p) > _mid[1] ) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys)>>1); + goto _match; + } + } + _trans += _klen; + } + +_match: + _trans = _json_indicies[_trans]; + cs = _json_trans_targs[_trans]; + + if ( _json_trans_actions[_trans] == 0 ) + goto _again; + + _acts = _json_actions + _json_trans_actions[_trans]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) + { + switch ( *_acts++ ) + { + case 1: +#line 2602 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 2: +#line 2604 "upb/json/parser.rl" + { p--; {stack[top++] = cs; cs = 23;goto _again;} } + break; + case 3: +#line 2608 "upb/json/parser.rl" + { start_text(parser, p); } + break; + case 4: +#line 2609 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_text(parser, p)); } + break; + case 5: +#line 2615 "upb/json/parser.rl" + { start_hex(parser); } + break; + case 6: +#line 2616 "upb/json/parser.rl" + { hexdigit(parser, p); } + break; + case 7: +#line 2617 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_hex(parser)); } + break; + case 8: +#line 2623 "upb/json/parser.rl" + { CHECK_RETURN_TOP(escape(parser, p)); } + break; + case 9: +#line 2629 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 10: +#line 2634 "upb/json/parser.rl" + { start_year(parser, p); } + break; + case 11: +#line 2635 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_year(parser, p)); } + break; + case 12: +#line 2639 "upb/json/parser.rl" + { start_month(parser, p); } + break; + case 13: +#line 2640 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_month(parser, p)); } + break; + case 14: +#line 2644 "upb/json/parser.rl" + { start_day(parser, p); } + break; + case 15: +#line 2645 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_day(parser, p)); } + break; + case 16: +#line 2649 "upb/json/parser.rl" + { start_hour(parser, p); } + break; + case 17: +#line 2650 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_hour(parser, p)); } + break; + case 18: +#line 2654 "upb/json/parser.rl" + { start_minute(parser, p); } + break; + case 19: +#line 2655 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_minute(parser, p)); } + break; + case 20: +#line 2659 "upb/json/parser.rl" + { start_second(parser, p); } + break; + case 21: +#line 2660 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_second(parser, p)); } + break; + case 22: +#line 2665 "upb/json/parser.rl" + { start_duration_base(parser, p); } + break; + case 23: +#line 2666 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_duration_base(parser, p)); } + break; + case 24: +#line 2668 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 25: +#line 2673 "upb/json/parser.rl" + { start_timestamp_base(parser); } + break; + case 26: +#line 2675 "upb/json/parser.rl" + { start_timestamp_fraction(parser, p); } + break; + case 27: +#line 2676 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_timestamp_fraction(parser, p)); } + break; + case 28: +#line 2678 "upb/json/parser.rl" + { start_timestamp_zone(parser, p); } + break; + case 29: +#line 2679 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_timestamp_zone(parser, p)); } + break; + case 30: +#line 2681 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 31: +#line 2686 "upb/json/parser.rl" + { start_fieldmask_path_text(parser, p); } + break; + case 32: +#line 2687 "upb/json/parser.rl" + { end_fieldmask_path_text(parser, p); } + break; + case 33: +#line 2692 "upb/json/parser.rl" + { start_fieldmask_path(parser); } + break; + case 34: +#line 2693 "upb/json/parser.rl" + { end_fieldmask_path(parser); } + break; + case 35: +#line 2699 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 36: +#line 2704 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { + {stack[top++] = cs; cs = 47;goto _again;} + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { + {stack[top++] = cs; cs = 40;goto _again;} + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_FIELDMASK)) { + {stack[top++] = cs; cs = 75;goto _again;} + } else { + {stack[top++] = cs; cs = 32;goto _again;} + } + } + break; + case 37: +#line 2717 "upb/json/parser.rl" + { p--; {stack[top++] = cs; cs = 78;goto _again;} } + break; + case 38: +#line 2722 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_member(parser, p); + } else { + start_member(parser); + } + } + break; + case 39: +#line 2729 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_membername(parser)); } + break; + case 40: +#line 2732 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + end_any_member(parser, p); + } else { + end_member(parser); + } + } + break; + case 41: +#line 2743 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_object(parser, p); + } else { + start_object(parser); + } + } + break; + case 42: +#line 2752 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + CHECK_RETURN_TOP(end_any_object(parser, p)); + } else { + end_object(parser); + } + } + break; + case 43: +#line 2764 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_array(parser)); } + break; + case 44: +#line 2768 "upb/json/parser.rl" + { end_array(parser); } + break; + case 45: +#line 2773 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_number(parser, p)); } + break; + case 46: +#line 2774 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_number(parser, p)); } + break; + case 47: +#line 2776 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_stringval(parser)); } + break; + case 48: +#line 2777 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_stringval(parser)); } + break; + case 49: +#line 2779 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, true)); } + break; + case 50: +#line 2781 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, false)); } + break; + case 51: +#line 2783 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_null(parser)); } + break; + case 52: +#line 2785 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_subobject_full(parser)); } + break; + case 53: +#line 2786 "upb/json/parser.rl" + { end_subobject_full(parser); } + break; + case 54: +#line 2791 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; +#line 3199 "upb/json/parser.c" + } + } + +_again: + if ( cs == 0 ) + goto _out; + if ( ++p != pe ) + goto _resume; + _test_eof: {} + if ( p == eof ) + { + const char *__acts = _json_actions + _json_eof_actions[cs]; + unsigned int __nacts = (unsigned int) *__acts++; + while ( __nacts-- > 0 ) { + switch ( *__acts++ ) { + case 0: +#line 2600 "upb/json/parser.rl" + { p--; {cs = stack[--top]; if ( p == pe ) + goto _test_eof; +goto _again;} } + break; + case 46: +#line 2774 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_number(parser, p)); } + break; + case 49: +#line 2779 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, true)); } + break; + case 50: +#line 2781 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, false)); } + break; + case 51: +#line 2783 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_null(parser)); } + break; + case 53: +#line 2786 "upb/json/parser.rl" + { end_subobject_full(parser); } + break; +#line 3241 "upb/json/parser.c" + } + } + } + + _out: {} + } + +#line 2819 "upb/json/parser.rl" + + if (p != pe) { + upb_status_seterrf(parser->status, "Parse error at '%.*s'\n", pe - p, p); + } else { + capture_suspend(parser, &p); + } + +error: + /* Save parsing state back to parser. */ + parser->current_state = cs; + parser->parser_top = top; + + return p - buf; +} + +static bool end(void *closure, const void *hd) { + upb_json_parser *parser = closure; + + /* Prevent compile warning on unused static constants. */ + UPB_UNUSED(json_start); + UPB_UNUSED(json_en_duration_machine); + UPB_UNUSED(json_en_fieldmask_machine); + UPB_UNUSED(json_en_number_machine); + UPB_UNUSED(json_en_string_machine); + UPB_UNUSED(json_en_timestamp_machine); + UPB_UNUSED(json_en_value_machine); + UPB_UNUSED(json_en_main); + + parse(parser, hd, &eof_ch, 0, NULL); + + return parser->current_state >= 106; +} + +static void json_parser_reset(upb_json_parser *p) { + int cs; + int top; + + p->top = p->stack; + init_frame(p->top); + + /* Emit Ragel initialization of the parser. */ + +#line 3292 "upb/json/parser.c" + { + cs = json_start; + top = 0; + } + +#line 2861 "upb/json/parser.rl" + p->current_state = cs; + p->parser_top = top; + accumulate_clear(p); + p->multipart_state = MULTIPART_INACTIVE; + p->capture = NULL; + p->accumulated = NULL; +} + +static upb_json_parsermethod *parsermethod_new(upb_json_codecache *c, + const upb_msgdef *md) { + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + upb_json_parsermethod *m = upb_malloc(alloc, sizeof(*m)); + + m->cache = c; + + upb_byteshandler_init(&m->input_handler_); + upb_byteshandler_setstring(&m->input_handler_, parse, m); + upb_byteshandler_setendstr(&m->input_handler_, end, m); + + upb_strtable_init2(&m->name_table, UPB_CTYPE_CONSTPTR, alloc); + + /* Build name_table */ + + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_value v = upb_value_constptr(f); + char *buf; + + /* Add an entry for the JSON name. */ + size_t len = upb_fielddef_getjsonname(f, NULL, 0); + buf = upb_malloc(alloc, len); + upb_fielddef_getjsonname(f, buf, len); + upb_strtable_insert3(&m->name_table, buf, strlen(buf), v, alloc); + + if (strcmp(buf, upb_fielddef_name(f)) != 0) { + /* Since the JSON name is different from the regular field name, add an + * entry for the raw name (compliant proto3 JSON parsers must accept + * both). */ + const char *name = upb_fielddef_name(f); + upb_strtable_insert3(&m->name_table, name, strlen(name), v, alloc); + } + } + + return m; +} + +/* Public API *****************************************************************/ + +upb_json_parser *upb_json_parser_create(upb_arena *arena, + const upb_json_parsermethod *method, + const upb_symtab* symtab, + upb_sink output, + upb_status *status, + bool ignore_json_unknown) { +#ifndef NDEBUG + const size_t size_before = upb_arena_bytesallocated(arena); +#endif + upb_json_parser *p = upb_arena_malloc(arena, sizeof(upb_json_parser)); + if (!p) return false; + + p->arena = arena; + p->method = method; + p->status = status; + p->limit = p->stack + UPB_JSON_MAX_DEPTH; + p->accumulate_buf = NULL; + p->accumulate_buf_size = 0; + upb_bytessink_reset(&p->input_, &method->input_handler_, p); + + json_parser_reset(p); + p->top->sink = output; + p->top->m = upb_handlers_msgdef(output.handlers); + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + set_name_table(p, p->top); + p->symtab = symtab; + + p->ignore_json_unknown = ignore_json_unknown; + + /* If this fails, uncomment and increase the value in parser.h. */ + /* fprintf(stderr, "%zd\n", upb_arena_bytesallocated(arena) - size_before); */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= + UPB_JSON_PARSER_SIZE); + return p; +} + +upb_bytessink upb_json_parser_input(upb_json_parser *p) { + return p->input_; +} + +const upb_byteshandler *upb_json_parsermethod_inputhandler( + const upb_json_parsermethod *m) { + return &m->input_handler_; +} + +upb_json_codecache *upb_json_codecache_new(void) { + upb_alloc *alloc; + upb_json_codecache *c; + + c = upb_gmalloc(sizeof(*c)); + + c->arena = upb_arena_new(); + alloc = upb_arena_alloc(c->arena); + + upb_inttable_init2(&c->methods, UPB_CTYPE_CONSTPTR, alloc); + + return c; +} + +void upb_json_codecache_free(upb_json_codecache *c) { + upb_arena_free(c->arena); + upb_gfree(c); +} + +const upb_json_parsermethod *upb_json_codecache_get(upb_json_codecache *c, + const upb_msgdef *md) { + upb_json_parsermethod *m; + upb_value v; + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + if (upb_inttable_lookupptr(&c->methods, md, &v)) { + return upb_value_getconstptr(v); + } + + m = parsermethod_new(c, md); + v = upb_value_constptr(m); + + if (!m) return NULL; + if (!upb_inttable_insertptr2(&c->methods, md, v, alloc)) return NULL; + + /* Populate parser methods for all submessages, so the name tables will + * be available during parsing. */ + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + + if (upb_fielddef_issubmsg(f)) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + const upb_json_parsermethod *sub_method = + upb_json_codecache_get(c, subdef); + + if (!sub_method) return NULL; + } + } + + return m; +} diff --git a/third_party/upb/kokoro/ubuntu/build.sh b/third_party/upb/kokoro/ubuntu/build.sh new file mode 100644 index 00000000000..ad8122b1bdb --- /dev/null +++ b/third_party/upb/kokoro/ubuntu/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Install the latest version of Bazel. +use_bazel.sh latest + +# Verify/query CMake +echo PATH=$PATH +ls -l `which cmake` +cmake --version + +# Log the bazel path and version. +which bazel +bazel version + +cd $(dirname $0)/../.. +bazel test --test_output=errors :all diff --git a/third_party/upb/kokoro/ubuntu/continuous.cfg b/third_party/upb/kokoro/ubuntu/continuous.cfg new file mode 100644 index 00000000000..fa97583b342 --- /dev/null +++ b/third_party/upb/kokoro/ubuntu/continuous.cfg @@ -0,0 +1,2 @@ +build_file: "upb/kokoro/ubuntu/build.sh" +timeout_mins: 15 diff --git a/third_party/upb/kokoro/ubuntu/presubmit.cfg b/third_party/upb/kokoro/ubuntu/presubmit.cfg new file mode 100644 index 00000000000..fa97583b342 --- /dev/null +++ b/third_party/upb/kokoro/ubuntu/presubmit.cfg @@ -0,0 +1,2 @@ +build_file: "upb/kokoro/ubuntu/build.sh" +timeout_mins: 15 diff --git a/third_party/upb/tests/benchmark.cc b/third_party/upb/tests/benchmark.cc new file mode 100644 index 00000000000..bcb4ec78b63 --- /dev/null +++ b/third_party/upb/tests/benchmark.cc @@ -0,0 +1,36 @@ + +#include +#include +#include "google/protobuf/descriptor.upb.h" +#include "google/protobuf/descriptor.upbdefs.h" + +upb_strview descriptor = google_protobuf_descriptor_proto_upbdefinit.descriptor; + +/* A buffer big enough to parse descriptor.proto without going to heap. */ +char buf[65535]; + +static void BM_CreateArena(benchmark::State& state) { + for (auto _ : state) { + upb_arena* arena = upb_arena_init(buf, sizeof(buf), NULL); + upb_arena_free(arena); + } +} +BENCHMARK(BM_CreateArena); + +static void BM_ParseDescriptor(benchmark::State& state) { + size_t bytes = 0; + for (auto _ : state) { + upb_arena* arena = upb_arena_init(buf, sizeof(buf), NULL); + google_protobuf_FileDescriptorProto* set = + google_protobuf_FileDescriptorProto_parse(descriptor.data, + descriptor.size, arena); + if (!set) { + printf("Failed to parse.\n"); + exit(1); + } + bytes += descriptor.size; + upb_arena_free(arena); + } + state.SetBytesProcessed(state.iterations() * descriptor.size); +} +BENCHMARK(BM_ParseDescriptor); diff --git a/third_party/upb/tests/bindings/googlepb/test_vs_proto2.cc b/third_party/upb/tests/bindings/googlepb/test_vs_proto2.cc new file mode 100644 index 00000000000..ac447e1b0cb --- /dev/null +++ b/third_party/upb/tests/bindings/googlepb/test_vs_proto2.cc @@ -0,0 +1,165 @@ +/* + * + * A test that verifies that our results are identical to proto2 for a + * given proto type and input protobuf. + */ + +#define __STDC_LIMIT_MACROS // So we get UINT32_MAX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tests/google_messages.pb.h" +#include "tests/upb_test.h" +#include "upb/bindings/googlepb/bridge.h" +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/pb/glue.h" +#include "upb/pb/varint.int.h" + +// Pull in string data from tests/google_message{1,2}.dat +// (the .h files are generated with xxd). +const unsigned char message1_data[] = { +#include "tests/google_message1.h" +}; + +const unsigned char message2_data[] = { +#include "tests/google_message2.h" +}; + +void compare_metadata(const google::protobuf::Descriptor* d, + const upb::MessageDef *upb_md) { + ASSERT(d->field_count() == upb_md->field_count()); + for (upb::MessageDef::const_field_iterator i = upb_md->field_begin(); + i != upb_md->field_end(); ++i) { + const upb::FieldDef* upb_f = *i; + const google::protobuf::FieldDescriptor *proto2_f = + d->FindFieldByNumber(upb_f->number()); + ASSERT(upb_f); + ASSERT(proto2_f); + ASSERT(upb_f->number() == (uint32_t)proto2_f->number()); + ASSERT(std::string(upb_f->name()) == proto2_f->name()); + ASSERT(upb_f->descriptor_type() == + static_cast(proto2_f->type())); + ASSERT(upb_f->IsSequence() == proto2_f->is_repeated()); + } +} + +void print_diff(const google::protobuf::Message& msg1, + const google::protobuf::Message& msg2) { + std::string text_str1; + std::string text_str2; + google::protobuf::TextFormat::PrintToString(msg1, &text_str1); + google::protobuf::TextFormat::PrintToString(msg2, &text_str2); + fprintf(stderr, "str1: %s, str2: %s\n", text_str1.c_str(), text_str2.c_str()); +} + +void parse_and_compare(google::protobuf::Message *msg1, + google::protobuf::Message *msg2, + const upb::Handlers *protomsg_handlers, + const char *str, size_t len, bool allow_jit) { + // Parse to both proto2 and upb. + ASSERT(msg1->ParseFromArray(str, len)); + + upb::pb::CodeCache cache; + ASSERT(cache.set_allow_jit(allow_jit)); + upb::reffed_ptr decoder_method( + cache.GetDecoderMethod(upb::pb::DecoderMethodOptions(protomsg_handlers))); + + upb::Status status; + upb::Environment env; + env.ReportErrorsTo(&status); + upb::Sink protomsg_sink(protomsg_handlers, msg2); + upb::pb::Decoder* decoder = + upb::pb::Decoder::Create(&env, decoder_method.get(), &protomsg_sink); + + msg2->Clear(); + bool ok = upb::BufferSource::PutBuffer(str, len, decoder->input()); + if (!ok) { + fprintf(stderr, "error parsing: %s\n", status.error_message()); + print_diff(*msg1, *msg2); + } + ASSERT(ok); + ASSERT(status.ok()); + + // Would like to just compare the message objects themselves, but + // unfortunately MessageDifferencer is not part of the open-source release of + // proto2, so we compare their serialized strings, which we expect will be + // equivalent. + std::string str1; + std::string str2; + msg1->SerializeToString(&str1); + msg2->SerializeToString(&str2); + if (str1 != str2) { + print_diff(*msg1, *msg2); + } + ASSERT(str1 == str2); + ASSERT(std::string(str, len) == str2); +} + +void test_zig_zag() { + for (uint64_t num = 5; num * 1.5 < UINT64_MAX; num *= 1.5) { + ASSERT(upb_zzenc_64(num) == + google::protobuf::internal::WireFormatLite::ZigZagEncode64(num)); + if (num < UINT32_MAX) { + ASSERT(upb_zzenc_32(num) == + google::protobuf::internal::WireFormatLite::ZigZagEncode32(num)); + } + } + +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + UPB_UNUSED(message1_data); + UPB_UNUSED(message2_data); + size_t len = sizeof(MESSAGE_DATA_IDENT); + const char *str = (const char*)MESSAGE_DATA_IDENT; + + MESSAGE_CIDENT msg1; + MESSAGE_CIDENT msg2; + + upb::reffed_ptr h( + upb::googlepb::WriteHandlers::New(msg1)); + + compare_metadata(msg1.GetDescriptor(), h->message_def()); + + // Run twice to test proper object reuse. + parse_and_compare(&msg1, &msg2, h.get(), str, len, false); + parse_and_compare(&msg1, &msg2, h.get(), str, len, true); + parse_and_compare(&msg1, &msg2, h.get(), str, len, false); + parse_and_compare(&msg1, &msg2, h.get(), str, len, true); + + // Test with DynamicMessage. + google::protobuf::DynamicMessageFactory* factory = + new google::protobuf::DynamicMessageFactory; + const google::protobuf::Message* prototype = + factory->GetPrototype(msg1.descriptor()); + google::protobuf::Message* dyn_msg1 = prototype->New(); + google::protobuf::Message* dyn_msg2 = prototype->New(); + h = upb::googlepb::WriteHandlers::New(*dyn_msg1); + parse_and_compare(dyn_msg1, dyn_msg2, h.get(), str, len, false); + parse_and_compare(dyn_msg1, dyn_msg2, h.get(), str, len, true); + delete dyn_msg1; + delete dyn_msg2; + delete factory; + + test_zig_zag(); + + printf("All tests passed, %d assertions.\n", num_assertions); + + google::protobuf::ShutdownProtobufLibrary(); + return 0; +} + +} diff --git a/third_party/upb/tests/bindings/lua/test_upb.lua b/third_party/upb/tests/bindings/lua/test_upb.lua new file mode 100644 index 00000000000..e4edda40032 --- /dev/null +++ b/third_party/upb/tests/bindings/lua/test_upb.lua @@ -0,0 +1,750 @@ + +local upb = require "upb" +local lunit = require "lunit" + +if _VERSION >= 'Lua 5.2' then + _ENV = lunit.module("testupb", "seeall") +else + module("testupb", lunit.testcase, package.seeall) +end + +function iter_to_array(iter) + local arr = {} + for v in iter do + arr[#arr + 1] = v + end + return arr +end + +function test_msgdef() + local f2 = upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32} + local o = upb.OneofDef{name = "field1", fields = {f2}} + local f = upb.FieldDef{name = "field3", number = 2, type = upb.TYPE_INT32} + + local m = upb.MessageDef{fields = {o, f}} + + assert_equal(f, m:lookup_name("field3")) + assert_equal(o, m:lookup_name("field1")) + assert_equal(f2, m:lookup_name("field2")) +end + +function test_fielddef() + local f = upb.FieldDef() + assert_false(f:is_frozen()) + assert_nil(f:number()) + assert_nil(f:name()) + assert_nil(f:type()) + assert_equal(upb.LABEL_OPTIONAL, f:label()) + + f:set_name("foo_field") + f:set_number(3) + f:set_label(upb.LABEL_REPEATED) + f:set_type(upb.TYPE_FLOAT) + + assert_equal("foo_field", f:name()) + assert_equal(3, f:number()) + assert_equal(upb.LABEL_REPEATED, f:label()) + assert_equal(upb.TYPE_FLOAT, f:type()) + + local f2 = upb.FieldDef{ + name = "foo", number = 5, type = upb.TYPE_DOUBLE, label = upb.LABEL_REQUIRED + } + + assert_equal("foo", f2:name()) + assert_equal(5, f2:number()) + assert_equal(upb.TYPE_DOUBLE, f2:type()) + assert_equal(upb.LABEL_REQUIRED, f2:label()) +end + +function test_enumdef() + local e = upb.EnumDef() + assert_equal(0, #e) + assert_nil(e:value(5)) + assert_nil(e:value("NONEXISTENT_NAME")) + + for name, value in e:values() do + fail() + end + + e:add("VAL1", 1) + e:add("VAL2", 2) + + local values = {} + for name, value in e:values() do + values[name] = value + end + + assert_equal(1, values["VAL1"]) + assert_equal(2, values["VAL2"]) + + local e2 = upb.EnumDef{ + values = { + {"FOO", 1}, + {"BAR", 77}, + } + } + + assert_equal(1, e2:value("FOO")) + assert_equal(77, e2:value("BAR")) + assert_equal("FOO", e2:value(1)) + assert_equal("BAR", e2:value(77)) + + e2:freeze() + + local f = upb.FieldDef{type = upb.TYPE_ENUM} + + -- No default set and no EnumDef to get a default from. + assert_equal(f:default(), nil) + + f:set_subdef(upb.EnumDef()) + -- No default to pull in from the EnumDef. + assert_equal(f:default(), nil) + + f:set_subdef(e2) + -- First member added to e2. + assert_equal(f:default(), "FOO") + + f:set_subdef(nil) + assert_equal(f:default(), nil) + + f:set_default(1) + assert_equal(f:default(), 1) + + f:set_default("YOYOYO") + assert_equal(f:default(), "YOYOYO") + + f:set_subdef(e2) + f:set_default(1) + -- It prefers to return a string, and could resolve the explicit "1" we set + -- it to to the string value. + assert_equal(f:default(), "FOO") + + -- FieldDef can specify default value by name or number, but the value must + -- exist at freeze time. + local m1 = upb.build_defs{ + upb.MessageDef{ + full_name = "A", + fields = { + upb.FieldDef{ + name = "f1", + number = 1, + type = upb.TYPE_ENUM, + subdef = e2, + default = "BAR" + }, + upb.FieldDef{ + name = "f2", + number = 2, + type = upb.TYPE_ENUM, + subdef = e2, + default = 77 + } + } + } + } + + assert_equal(m1:field("f1"):default(), "BAR") + assert_equal(m1:field("f1"):default(), "BAR") + + assert_error_match( + "enum default for field A.f1 .DOESNT_EXIST. is not in the enum", + function() + local m1 = upb.build_defs{ + upb.MessageDef{ + full_name = "A", + fields = { + upb.FieldDef{ + name = "f1", + number = 1, + type = upb.TYPE_ENUM, + subdef = e2, + default = "DOESNT_EXIST" + } + } + } + } + end + ) + + assert_error_match( + "enum default for field A.f1 .142. is not in the enum", + function() + local m1 = upb.build_defs{ + upb.MessageDef{ + full_name = "A", + fields = { + upb.FieldDef{ + name = "f1", + number = 1, + type = upb.TYPE_ENUM, + subdef = e2, + default = 142 + } + } + } + } + end + ) +end + +function test_empty_msgdef() + local md = upb.MessageDef() + assert_nil(md:full_name()) -- Def without name is anonymous. + assert_false(md:is_frozen()) + assert_equal(0, #md) + assert_nil(md:field("nonexistent_field")) + assert_nil(md:field(3)) + for field in md:fields() do + fail() + end + + upb.freeze(md) + assert_true(md:is_frozen()) + assert_equal(0, #md) + assert_nil(md:field("nonexistent_field")) + assert_nil(md:field(3)) + for field in md:fields() do + fail() + end +end + +function test_msgdef_constructor() + local f1 = upb.FieldDef{name = "field1", number = 7, type = upb.TYPE_INT32} + local f2 = upb.FieldDef{name = "field2", number = 8, type = upb.TYPE_INT32} + local md = upb.MessageDef{ + full_name = "TestMessage", + fields = {f1, f2} + } + assert_equal("TestMessage", md:full_name()) + assert_false(md:is_frozen()) + assert_equal(2, #md) + assert_equal(f1, md:field("field1")) + assert_equal(f2, md:field("field2")) + assert_equal(f1, md:field(7)) + assert_equal(f2, md:field(8)) + local count = 0 + local found = {} + for field in md:fields() do + count = count + 1 + found[field] = true + end + assert_equal(2, count) + assert_true(found[f1]) + assert_true(found[f2]) + + upb.freeze(md) +end + +function test_iteration() + -- Test that we cannot crash the process even if we modify the set of fields + -- during iteration. + local md = upb.MessageDef{full_name = "TestMessage"} + + for i=1,10 do + md:add(upb.FieldDef{ + name = "field" .. tostring(i), + number = 1000 - i, + type = upb.TYPE_INT32 + }) + end + + local add = #md + for f in md:fields() do + if add > 0 then + add = add - 1 + for i=10000,11000 do + local field_name = "field" .. tostring(i) + -- We want to add fields to the table to trigger a table resize, + -- but we must skip it if the field name or number already exists + -- otherwise it will raise an error. + if md:field(field_name) == nil and + md:field(i) == nil then + md:add(upb.FieldDef{ + name = field_name, + number = i, + type = upb.TYPE_INT32 + }) + end + end + end + end + + -- Test that iterators don't crash the process even if the MessageDef goes + -- out of scope. + -- + -- Note: have previously verified that this can indeed crash the process if + -- we do not explicitly add a reference from the iterator to the underlying + -- MessageDef. + local iter = md:fields() + md = nil + collectgarbage() + while iter() do + end + + local ed = upb.EnumDef{ + values = { + {"FOO", 1}, + {"BAR", 77}, + } + } + iter = ed:values() + ed = nil + collectgarbage() + while iter() do + end +end + +function test_msgdef_setters() + local md = upb.MessageDef() + md:set_full_name("Message1") + assert_equal("Message1", md:full_name()) + local f = upb.FieldDef{name = "field1", number = 3, type = upb.TYPE_DOUBLE} + md:add(f) + assert_equal(1, #md) + assert_equal(f, md:field("field1")) +end + +function test_msgdef_errors() + assert_error(function() upb.MessageDef{bad_initializer_key = 5} end) + local md = upb.MessageDef() + assert_error(function() + -- Duplicate field number. + upb.MessageDef{ + fields = { + upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32} + } + } + end) + assert_error(function() + -- Duplicate field name. + upb.MessageDef{ + fields = { + upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "field1", number = 2, type = upb.TYPE_INT32} + } + } + end) + + assert_error(function() + -- Duplicate field name. + upb.MessageDef{ + fields = { + upb.OneofDef{name = "field1", fields = { + upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32}, + }}, + upb.FieldDef{name = "field2", number = 2, type = upb.TYPE_INT32} + } + } + end) + + -- attempt to set a name with embedded NULLs. + assert_error_match("names cannot have embedded NULLs", function() + md:set_full_name("abc\0def") + end) + + upb.freeze(md) + -- Attempt to mutate frozen MessageDef. + assert_error_match("frozen", function() + md:add(upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}) + end) + assert_error_match("frozen", function() + md:set_full_name("abc") + end) + + -- Attempt to freeze a msgdef without freezing its subdef. + assert_error_match("is not frozen or being frozen", function() + m1 = upb.MessageDef() + upb.freeze( + upb.MessageDef{ + fields = { + upb.FieldDef{name = "f1", number = 1, type = upb.TYPE_MESSAGE, + subdef = m1} + } + } + ) + end) +end + +function test_symtab() + local empty = upb.SymbolTable() + assert_equal(0, #iter_to_array(empty:defs(upb.DEF_ANY))) + assert_equal(0, #iter_to_array(empty:defs(upb.DEF_MSG))) + assert_equal(0, #iter_to_array(empty:defs(upb.DEF_ENUM))) + + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage"}, + upb.MessageDef{full_name = "ContainingMessage", fields = { + upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "field2", number = 2, type = upb.TYPE_MESSAGE, + subdef_name = ".TestMessage"} + } + } + } + + local msgdef1 = symtab:lookup("TestMessage") + local msgdef2 = symtab:lookup("ContainingMessage") + assert_not_nil(msgdef1) + assert_not_nil(msgdef2) + assert_equal(msgdef1, msgdef2:field("field2"):subdef()) + assert_true(msgdef1:is_frozen()) + assert_true(msgdef2:is_frozen()) + + symtab:add{ + upb.MessageDef{full_name = "ContainingMessage2", fields = { + upb.FieldDef{name = "field5", number = 5, type = upb.TYPE_MESSAGE, + subdef = msgdef2} + } + } + } + + local msgdef3 = symtab:lookup("ContainingMessage2") + assert_not_nil(msgdef3) + assert_equal(msgdef3:field("field5"):subdef(), msgdef2) +end + +function test_numeric_array() + local function test_for_numeric_type(upb_type, val, too_big, too_small, bad3) + local array = upb.Array(upb_type) + assert_equal(0, #array) + + -- 0 is never a valid index in Lua. + assert_error_match("array index", function() return array[0] end) + -- Past the end of the array. + assert_error_match("array index", function() return array[1] end) + + array[1] = val + assert_equal(val, array[1]) + assert_equal(1, #array) + assert_equal(val, array[1]) + -- Past the end of the array. + assert_error_match("array index", function() return array[2] end) + + array[2] = 10 + assert_equal(val, array[1]) + assert_equal(10, array[2]) + assert_equal(2, #array) + -- Past the end of the array. + assert_error_match("array index", function() return array[3] end) + + local n = 1 + for i, val in upb.ipairs(array) do + assert_equal(n, i) + n = n + 1 + assert_equal(array[i], val) + end + + -- Values that are out of range. + local errmsg = "not an integer or out of range" + if too_small then + assert_error_match(errmsg, function() array[3] = too_small end) + end + if too_big then + assert_error_match(errmsg, function() array[3] = too_big end) + end + if bad3 then + assert_error_match(errmsg, function() array[3] = bad3 end) + end + + -- Can't assign other Lua types. + errmsg = "bad argument #3" + assert_error_match(errmsg, function() array[3] = "abc" end) + assert_error_match(errmsg, function() array[3] = true end) + assert_error_match(errmsg, function() array[3] = false end) + assert_error_match(errmsg, function() array[3] = nil end) + assert_error_match(errmsg, function() array[3] = {} end) + assert_error_match(errmsg, function() array[3] = print end) + assert_error_match(errmsg, function() array[3] = array end) + end + + -- in-range of 64-bit types but not exactly representable as double + local bad64 = 2^68 - 1 + + test_for_numeric_type(upb.TYPE_UINT32, 2^32 - 1, 2^32, -1, 5.1) + test_for_numeric_type(upb.TYPE_UINT64, 2^63, 2^64, -1, bad64) + test_for_numeric_type(upb.TYPE_INT32, 2^31 - 1, 2^31, -2^31 - 1, 5.1) + -- Enums don't exist at a language level in Lua, so we just represent enum + -- values as int32s. + test_for_numeric_type(upb.TYPE_ENUM, 2^31 - 1, 2^31, -2^31 - 1, 5.1) + test_for_numeric_type(upb.TYPE_INT64, 2^62, 2^63, -2^64, bad64) + test_for_numeric_type(upb.TYPE_FLOAT, 340282306073709652508363335590014353408) + test_for_numeric_type(upb.TYPE_DOUBLE, 10^101) +end + +function test_string_array() + local function test_for_string_type(upb_type) + local array = upb.Array(upb_type) + assert_equal(0, #array) + + -- 0 is never a valid index in Lua. + assert_error_match("array index", function() return array[0] end) + -- Past the end of the array. + assert_error_match("array index", function() return array[1] end) + + array[1] = "foo" + assert_equal("foo", array[1]) + assert_equal(1, #array) + -- Past the end of the array. + assert_error_match("array index", function() return array[2] end) + + local array2 = upb.Array(upb_type) + assert_equal(0, #array2) + + array[2] = "bar" + assert_equal("foo", array[1]) + assert_equal("bar", array[2]) + assert_equal(2, #array) + -- Past the end of the array. + assert_error_match("array index", function() return array[3] end) + + local n = 1 + for i, val in upb.ipairs(array) do + assert_equal(n, i) + n = n + 1 + assert_equal(array[i], val) + end + assert_equal(3, n) + + -- Can't assign other Lua types. + assert_error_match("Expected string", function() array[3] = 123 end) + assert_error_match("Expected string", function() array[3] = true end) + assert_error_match("Expected string", function() array[3] = false end) + assert_error_match("Expected string", function() array[3] = nil end) + assert_error_match("Expected string", function() array[3] = {} end) + assert_error_match("Expected string", function() array[3] = print end) + assert_error_match("Expected string", function() array[3] = array end) + end + + test_for_string_type(upb.TYPE_STRING) + test_for_string_type(upb.TYPE_BYTES) +end + +function test_msg_primitives() + local function test_for_numeric_type(upb_type, val, too_big, too_small, bad3) + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "f", number = 1, type = upb_type}, + } + } + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + msg = TestMessage() + + -- Defaults to zero + assert_equal(0, msg.f) + + msg.f = 0 + assert_equal(0, msg.f) + + msg.f = val + assert_equal(val, msg.f) + + local errmsg = "not an integer or out of range" + if too_small then + assert_error_match(errmsg, function() msg.f = too_small end) + end + if too_big then + assert_error_match(errmsg, function() msg.f = too_big end) + end + if bad3 then + assert_error_match(errmsg, function() msg.f = bad3 end) + end + + -- Can't assign other Lua types. + errmsg = "bad argument #3" + assert_error_match(errmsg, function() msg.f = "abc" end) + assert_error_match(errmsg, function() msg.f = true end) + assert_error_match(errmsg, function() msg.f = false end) + assert_error_match(errmsg, function() msg.f = nil end) + assert_error_match(errmsg, function() msg.f = {} end) + assert_error_match(errmsg, function() msg.f = print end) + assert_error_match(errmsg, function() msg.f = array end) + end + + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{ + name = "i32", number = 1, type = upb.TYPE_INT32, default = 1}, + upb.FieldDef{ + name = "u32", number = 2, type = upb.TYPE_UINT32, default = 2}, + upb.FieldDef{ + name = "i64", number = 3, type = upb.TYPE_INT64, default = 3}, + upb.FieldDef{ + name = "u64", number = 4, type = upb.TYPE_UINT64, default = 4}, + upb.FieldDef{ + name = "dbl", number = 5, type = upb.TYPE_DOUBLE, default = 5}, + upb.FieldDef{ + name = "flt", number = 6, type = upb.TYPE_FLOAT, default = 6}, + upb.FieldDef{ + name = "bool", number = 7, type = upb.TYPE_BOOL, default = true}, + } + } + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + msg = TestMessage() + + -- Unset member returns default value. + -- TODO(haberman): re-enable these when we have descriptor-based reflection. + -- assert_equal(1, msg.i32) + -- assert_equal(2, msg.u32) + -- assert_equal(3, msg.i64) + -- assert_equal(4, msg.u64) + -- assert_equal(5, msg.dbl) + -- assert_equal(6, msg.flt) + -- assert_equal(true, msg.bool) + + -- Attempts to access non-existent fields fail. + assert_error_match("no such field", function() msg.no_such = 1 end) + + msg.i32 = 10 + msg.u32 = 20 + msg.i64 = 30 + msg.u64 = 40 + msg.dbl = 50 + msg.flt = 60 + msg.bool = true + + assert_equal(10, msg.i32) + assert_equal(20, msg.u32) + assert_equal(30, msg.i64) + assert_equal(40, msg.u64) + assert_equal(50, msg.dbl) + assert_equal(60, msg.flt) + assert_equal(true, msg.bool) + + test_for_numeric_type(upb.TYPE_UINT32, 2^32 - 1, 2^32, -1, 5.1) + test_for_numeric_type(upb.TYPE_UINT64, 2^62, 2^64, -1, bad64) + test_for_numeric_type(upb.TYPE_INT32, 2^31 - 1, 2^31, -2^31 - 1, 5.1) + test_for_numeric_type(upb.TYPE_INT64, 2^61, 2^63, -2^64, bad64) + test_for_numeric_type(upb.TYPE_FLOAT, 2^20) + test_for_numeric_type(upb.TYPE_DOUBLE, 10^101) +end + +function test_msg_array() + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "i32_array", number = 1, type = upb.TYPE_INT32, + label = upb.LABEL_REPEATED}, + } + } + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + msg = TestMessage() + + assert_nil(msg.i32_array) + + -- Can't assign a scalar; array is expected. + assert_error_match("lupb.array expected", function() msg.i32_array = 5 end) + + -- Can't assign array of the wrong type. + local function assign_int64() + msg.i32_array = upb.Array(upb.TYPE_INT64) + end + assert_error_match("Array had incorrect type", assign_int64) + + local arr = upb.Array(upb.TYPE_INT32) + msg.i32_array = arr + assert_equal(arr, msg.i32_array) + + -- Can't assign other Lua types. + assert_error_match("array expected", function() msg.i32_array = "abc" end) + assert_error_match("array expected", function() msg.i32_array = true end) + assert_error_match("array expected", function() msg.i32_array = false end) + assert_error_match("array expected", function() msg.i32_array = nil end) + assert_error_match("array expected", function() msg.i32_array = {} end) + assert_error_match("array expected", function() msg.i32_array = print end) +end + +function test_msg_submsg() + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "submsg", number = 1, type = upb.TYPE_MESSAGE, + subdef_name = ".SubMessage"}, + } + }, + upb.MessageDef{full_name = "SubMessage"} + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + SubMessage = factory:get_message_class("SubMessage") + msg = TestMessage() + + assert_nil(msg.submsg) + + -- Can't assign message of the wrong type. + local function assign_int64() + msg.submsg = TestMessage() + end + assert_error_match("Message had incorrect type", assign_int64) + + local sub = SubMessage() + msg.submsg = sub + assert_equal(sub, msg.submsg) + + -- Can't assign other Lua types. + assert_error_match("msg expected", function() msg.submsg = "abc" end) + assert_error_match("msg expected", function() msg.submsg = true end) + assert_error_match("msg expected", function() msg.submsg = false end) + assert_error_match("msg expected", function() msg.submsg = nil end) + assert_error_match("msg expected", function() msg.submsg = {} end) + assert_error_match("msg expected", function() msg.submsg = print end) +end + +-- Lua 5.1 and 5.2 have slightly different semantics for how a finalizer +-- can be defined in Lua. +if _VERSION >= 'Lua 5.2' then + function defer(fn) + setmetatable({}, { __gc = fn }) + end +else + function defer(fn) + getmetatable(newproxy(true)).__gc = fn + end +end + +function test_finalizer() + -- Tests that we correctly handle a call into an already-finalized object. + -- Collectible objects are finalized in the opposite order of creation. + do + local t = {} + defer(function() + assert_error_match("called into dead object", function() + -- Generic def call. + t[1]:full_name() + end) + assert_error_match("called into dead object", function() + -- Specific msgdef call. + t[1]:add() + end) + assert_error_match("called into dead object", function() + t[2]:values() + end) + assert_error_match("called into dead object", function() + t[3]:number() + end) + assert_error_match("called into dead object", function() + t[4]:lookup() + end) + end) + t = { + upb.MessageDef(), + upb.EnumDef(), + upb.FieldDef(), + upb.SymbolTable(), + } + end + collectgarbage() +end + +local stats = lunit.main() + +if stats.failed > 0 or stats.errors > 0 then + error("One or more errors in test suite") +end diff --git a/third_party/upb/tests/bindings/lua/test_upb.pb.lua b/third_party/upb/tests/bindings/lua/test_upb.pb.lua new file mode 100644 index 00000000000..ea6de099895 --- /dev/null +++ b/third_party/upb/tests/bindings/lua/test_upb.pb.lua @@ -0,0 +1,80 @@ + +-- Require "pb" first to ensure that the transitive require of "upb" is +-- handled properly by the "pb" module. +local pb = require "upb.pb" +local upb = require "upb" +local lunit = require "lunit" + +if _VERSION >= 'Lua 5.2' then + _ENV = lunit.module("testupb_pb", "seeall") +else + module("testupb_pb", lunit.testcase, package.seeall) +end + +local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "i32", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "u32", number = 2, type = upb.TYPE_UINT32}, + upb.FieldDef{name = "i64", number = 3, type = upb.TYPE_INT64}, + upb.FieldDef{name = "u64", number = 4, type = upb.TYPE_UINT64}, + upb.FieldDef{name = "dbl", number = 5, type = upb.TYPE_DOUBLE}, + upb.FieldDef{name = "flt", number = 6, type = upb.TYPE_FLOAT}, + upb.FieldDef{name = "bool", number = 7, type = upb.TYPE_BOOL}, + } + } +} + +local factory = upb.MessageFactory(symtab); +local TestMessage = factory:get_message_class("TestMessage") + +function test_parse_primitive() + local binary_pb = + "\008\128\128\128\128\002\016\128\128\128\128\004\024\128\128" + .. "\128\128\128\128\128\002\032\128\128\128\128\128\128\128\001\041\000" + .. "\000\000\000\000\000\248\063\053\000\000\096\064\056\001" + local msg = TestMessage() + pb.decode(msg, binary_pb) + assert_equal(536870912, msg.i32) + assert_equal(1073741824, msg.u32) + assert_equal(1125899906842624, msg.i64) + assert_equal(562949953421312, msg.u64) + assert_equal(1.5, msg.dbl) + assert_equal(3.5, msg.flt) + assert_equal(true, msg.bool) + + local encoded = pb.encode(msg) + local msg2 = TestMessage() + pb.decode(msg2, encoded) + assert_equal(536870912, msg.i32) + assert_equal(1073741824, msg.u32) + assert_equal(1125899906842624, msg.i64) + assert_equal(562949953421312, msg.u64) + assert_equal(1.5, msg.dbl) + assert_equal(3.5, msg.flt) + assert_equal(true, msg.bool) +end + +function test_parse_string() + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "str", number = 1, type = upb.TYPE_STRING}, + } + } + } + + local factory = upb.MessageFactory(symtab); + local TestMessage = factory:get_message_class("TestMessage") + + local binary_pb = "\010\005Hello" + msg = TestMessage() + pb.decode(msg, binary_pb) + -- TODO(haberman): re-enable when this stuff works better. + -- assert_equal("Hello", msg.str) +end + + +local stats = lunit.main() + +if stats.failed > 0 or stats.errors > 0 then + error("One or more errors in test suite") +end diff --git a/third_party/upb/tests/bindings/ruby/upb.rb b/third_party/upb/tests/bindings/ruby/upb.rb new file mode 100644 index 00000000000..3e06c1798c6 --- /dev/null +++ b/third_party/upb/tests/bindings/ruby/upb.rb @@ -0,0 +1,62 @@ +#!/usr/bin/ruby +# +# Tests for Ruby upb extension. + +require 'test/unit' +require 'set' +require 'upb' + +def get_descriptor + File.open("upb/descriptor/descriptor.pb").read +end + +def load_descriptor + symtab = Upb::SymbolTable.new + symtab.load_descriptor(get_descriptor()) + return symtab +end + +def get_message_class(name) + return Upb.get_message_class(load_descriptor().lookup(name)) +end + +class TestRubyExtension < Test::Unit::TestCase + def test_parsedescriptor + msgdef = load_descriptor.lookup("google.protobuf.FileDescriptorSet") + assert_instance_of(Upb::MessageDef, msgdef) + + file_descriptor_set = Upb.get_message_class(msgdef) + msg = file_descriptor_set.parse(get_descriptor()) + + # A couple message types we know should exist. + names = Set.new(["DescriptorProto", "FieldDescriptorProto"]) + + msg.file.each { |file| + file.message_type.each { |message_type| + names.delete(message_type.name) + } + } + + assert_equal(0, names.size) + end + + def test_parseserialize + field_descriptor_proto = get_message_class("google.protobuf.FieldDescriptorProto") + field_options = get_message_class("google.protobuf.FieldOptions") + + field = field_descriptor_proto.new + + field.name = "MyName" + field.number = 5 + field.options = field_options.new + field.options.packed = true + + serialized = Upb::Message.serialize(field) + + field2 = field_descriptor_proto.parse(serialized) + + assert_equal("MyName", field2.name) + assert_equal(5, field2.number) + assert_equal(true, field2.options.packed) + end +end diff --git a/third_party/upb/tests/conformance_upb.c b/third_party/upb/tests/conformance_upb.c new file mode 100644 index 00000000000..6063c9941ed --- /dev/null +++ b/third_party/upb/tests/conformance_upb.c @@ -0,0 +1,179 @@ +/* This is a upb implementation of the upb conformance tests, see: + * https://github.com/google/protobuf/tree/master/conformance + */ + +#include +#include +#include +#include +#include + +#include "conformance/conformance.upb.h" +#include "src/google/protobuf/test_messages_proto3.upb.h" + +int test_count = 0; + +bool CheckedRead(int fd, void *buf, size_t len) { + size_t ofs = 0; + while (len > 0) { + ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + + if (bytes_read == 0) return false; + + if (bytes_read < 0) { + perror("reading from test runner"); + exit(1); + } + + len -= bytes_read; + ofs += bytes_read; + } + + return true; +} + +void CheckedWrite(int fd, const void *buf, size_t len) { + if ((size_t)write(fd, buf, len) != len) { + perror("writing to test runner"); + exit(1); + } +} + +bool strview_eql(upb_strview view, const char *str) { + return view.size == strlen(str) && memcmp(view.data, str, view.size) == 0; +} + +static const char *proto3_msg = + "protobuf_test_messages.proto3.TestAllTypesProto3"; + +void DoTest( + const conformance_ConformanceRequest* request, + conformance_ConformanceResponse *response, + upb_arena *arena) { + protobuf_test_messages_proto3_TestAllTypesProto3 *test_message; + + if (!strview_eql(conformance_ConformanceRequest_message_type(request), + proto3_msg)) { + static const char msg[] = "Only proto3 for now."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + + switch (conformance_ConformanceRequest_payload_case(request)) { + case conformance_ConformanceRequest_payload_protobuf_payload: { + upb_strview payload = conformance_ConformanceRequest_protobuf_payload(request); + test_message = protobuf_test_messages_proto3_TestAllTypesProto3_parse( + payload.data, payload.size, arena); + + if (!test_message) { + static const char msg[] = "Parse error"; + conformance_ConformanceResponse_set_parse_error( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + break; + } + + case conformance_ConformanceRequest_payload_NOT_SET: + fprintf(stderr, "conformance_upb: Request didn't have payload.\n"); + return; + + default: { + static const char msg[] = "Unsupported input format."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + } + + switch (conformance_ConformanceRequest_requested_output_format(request)) { + case conformance_UNSPECIFIED: + fprintf(stderr, "conformance_upb: Unspecified output format.\n"); + exit(1); + + case conformance_PROTOBUF: { + size_t serialized_len; + char *serialized = + protobuf_test_messages_proto3_TestAllTypesProto3_serialize( + test_message, arena, &serialized_len); + if (!serialized) { + static const char msg[] = "Error serializing."; + conformance_ConformanceResponse_set_serialize_error( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + conformance_ConformanceResponse_set_protobuf_payload( + response, upb_strview_make(serialized, serialized_len)); + break; + } + + default: { + static const char msg[] = "Unsupported output format."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + } + + return; +} + +bool DoTestIo(void) { + upb_arena *arena; + upb_alloc *alloc; + upb_status status; + char *serialized_input; + char *serialized_output; + uint32_t input_size; + size_t output_size; + conformance_ConformanceRequest *request; + conformance_ConformanceResponse *response; + + if (!CheckedRead(STDIN_FILENO, &input_size, sizeof(uint32_t))) { + /* EOF. */ + return false; + } + + arena = upb_arena_new(); + alloc = upb_arena_alloc(arena); + serialized_input = upb_malloc(alloc, input_size); + + if (!CheckedRead(STDIN_FILENO, serialized_input, input_size)) { + fprintf(stderr, "conformance_upb: unexpected EOF on stdin.\n"); + exit(1); + } + + request = + conformance_ConformanceRequest_parse(serialized_input, input_size, arena); + response = conformance_ConformanceResponse_new(arena); + + if (request) { + DoTest(request, response, arena); + } else { + fprintf(stderr, "conformance_upb: parse of ConformanceRequest failed: %s\n", + upb_status_errmsg(&status)); + } + + serialized_output = conformance_ConformanceResponse_serialize( + response, arena, &output_size); + + CheckedWrite(STDOUT_FILENO, &output_size, sizeof(uint32_t)); + CheckedWrite(STDOUT_FILENO, serialized_output, output_size); + + test_count++; + + upb_arena_free(arena); + + return true; +} + +int main(void) { + while (1) { + if (!DoTestIo()) { + fprintf(stderr, "conformance_upb: received EOF from test runner " + "after %d tests, exiting\n", test_count); + return 0; + } + } +} diff --git a/third_party/upb/tests/conformance_upb_failures.txt b/third_party/upb/tests/conformance_upb_failures.txt new file mode 100644 index 00000000000..05aab585e77 --- /dev/null +++ b/third_party/upb/tests/conformance_upb_failures.txt @@ -0,0 +1 @@ +Required.ProtobufInput.PrematureEofInSubmessageValue.MESSAGE diff --git a/third_party/upb/tests/corpus/README b/third_party/upb/tests/corpus/README new file mode 100644 index 00000000000..9bd8f1e93d6 --- /dev/null +++ b/third_party/upb/tests/corpus/README @@ -0,0 +1 @@ +Corpus folder for fuzzing diff --git a/third_party/upb/tests/corpus/temp.cc b/third_party/upb/tests/corpus/temp.cc new file mode 100644 index 00000000000..2bf1160ce69 --- /dev/null +++ b/third_party/upb/tests/corpus/temp.cc @@ -0,0 +1 @@ +// Hello World diff --git a/third_party/upb/tests/file_descriptor_parsenew_fuzzer.cc b/third_party/upb/tests/file_descriptor_parsenew_fuzzer.cc new file mode 100644 index 00000000000..057e62d8bf8 --- /dev/null +++ b/third_party/upb/tests/file_descriptor_parsenew_fuzzer.cc @@ -0,0 +1,15 @@ +#include + +#include "google/protobuf/descriptor.upb.h" +#include "upb/upb.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + upb::Arena arena; + google_protobuf_FileDescriptorProto_parse(reinterpret_cast(data), + size, arena.ptr()); + return 0; +} + +#ifndef HAVE_FUZZER +int main() {} +#endif diff --git a/third_party/upb/tests/google_message1.dat b/third_party/upb/tests/google_message1.dat new file mode 100644 index 00000000000..bc0f064cc2e Binary files /dev/null and b/third_party/upb/tests/google_message1.dat differ diff --git a/third_party/upb/tests/google_message2.dat b/third_party/upb/tests/google_message2.dat new file mode 100644 index 00000000000..06c09441b9e Binary files /dev/null and b/third_party/upb/tests/google_message2.dat differ diff --git a/third_party/upb/tests/google_messages.proto b/third_party/upb/tests/google_messages.proto new file mode 100644 index 00000000000..489c4707327 --- /dev/null +++ b/third_party/upb/tests/google_messages.proto @@ -0,0 +1,149 @@ + +package benchmarks; + +option optimize_for = SPEED; + +enum Foo { + FOO_VALUE = 1; + FOO_VALUE2 = 2; +} + +message Simple { + message M2 { + optional int32 f1 = 1234567; + } + optional M2 m2 = 1; +} + +message SpeedMessage1 { + required string field1 = 1; + optional string field9 = 9; + optional string field18 = 18; + optional bool field80 = 80 [default=false]; + optional bool field81 = 81 [default=true]; + required int32 field2 = 2; + required int32 field3 = 3; + optional int32 field280 = 280; + optional int32 field6 = 6 [default=0]; + optional int64 field22 = 22; + optional string field4 = 4; + repeated fixed64 field5 = 5; + optional bool field59 = 59 [default=false]; + optional string field7 = 7; + optional int32 field16 = 16; + optional int32 field130 = 130 [default=0]; + optional bool field12 = 12 [default=true]; + optional bool field17 = 17 [default=true]; + optional bool field13 = 13 [default=true]; + optional bool field14 = 14 [default=true]; + optional int32 field104 = 104 [default=0]; + optional int32 field100 = 100 [default=0]; + optional int32 field101 = 101 [default=0]; + optional string field102 = 102; + optional string field103 = 103; + optional int32 field29 = 29 [default=0]; + optional bool field30 = 30 [default=false]; + optional int32 field60 = 60 [default=-1]; + optional int32 field271 = 271 [default=-1]; + optional int32 field272 = 272 [default=-1]; + optional int32 field150 = 150; + optional int32 field23 = 23 [default=0]; + optional bool field24 = 24 [default=false]; + optional int32 field25 = 25 [default=0]; + optional SpeedMessage1SubMessage field15 = 15; + optional bool field78 = 78; + optional int32 field67 = 67 [default=0]; + optional int32 field68 = 68; + optional int32 field128 = 128 [default=0]; + optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"]; + optional int32 field131 = 131 [default=0]; + optional Foo field132 = 132 [default=FOO_VALUE]; +} + +message SpeedMessage1SubMessage { + optional int32 field1 = 1 [default=0]; + optional int32 field2 = 2 [default=0]; + optional int32 field3 = 3 [default=0]; + optional string field15 = 15 [default="FOOBAR!"]; + optional bool field12 = 12 [default=true]; + optional int64 field13 = 13; + optional int64 field14 = 14; + optional int32 field16 = 16; + optional int32 field19 = 19 [default=2]; + optional bool field20 = 20 [default=true]; + optional bool field28 = 28 [default=true]; + optional fixed64 field21 = 21; + optional int32 field22 = 22; + optional bool field23 = 23 [ default=false ]; + optional bool field206 = 206 [default=false]; + optional fixed32 field203 = 203; + optional int32 field204 = 204; + optional string field205 = 205; + optional uint64 field207 = 207; + optional uint64 field300 = 300; +} + +message SpeedMessage2 { + optional string field1 = 1; + optional int64 field3 = 3; + optional int64 field4 = 4; + optional int64 field30 = 30; + optional bool field75 = 75 [default=false]; + optional string field6 = 6; + optional bytes field2 = 2; + optional int32 field21 = 21 [default=0]; + optional int32 field71 = 71; + optional float field25 = 25; + optional int32 field109 = 109 [default=0]; + optional int32 field210 = 210 [default=0]; + optional int32 field211 = 211 [default=0]; + optional int32 field212 = 212 [default=0]; + optional int32 field213 = 213 [default=0]; + optional int32 field216 = 216 [default=0]; + optional int32 field217 = 217 [default=0]; + optional int32 field218 = 218 [default=0]; + optional int32 field220 = 220 [default=0]; + optional int32 field221 = 221 [default=0]; + optional float field222 = 222 [default=0.0]; + optional int32 field63 = 63; + + repeated group Group1 = 10 { + required float field11 = 11; + optional float field26 = 26; + optional string field12 = 12; + optional string field13 = 13; + repeated string field14 = 14; + required uint64 field15 = 15; + optional int32 field5 = 5; + optional string field27 = 27; + optional int32 field28 = 28; + optional string field29 = 29; + optional string field16 = 16; + repeated string field22 = 22; + repeated int32 field73 = 73; + optional int32 field20 = 20 [default=0]; + optional string field24 = 24; + optional SpeedMessage2GroupedMessage field31 = 31; + } + repeated string field128 = 128; + optional int64 field131 = 131; + repeated string field127 = 127; + optional int32 field129 = 129; + repeated int64 field130 = 130; + optional bool field205 = 205 [default=false]; + optional bool field206 = 206 [default=false]; +} + +message SpeedMessage2GroupedMessage { + optional float field1 = 1; + optional float field2 = 2; + optional float field3 = 3 [default=0.0]; + optional bool field4 = 4; + optional bool field5 = 5; + optional bool field6 = 6 [default=true]; + optional bool field7 = 7 [default=false]; + optional float field8 = 8; + optional bool field9 = 9; + optional float field10 = 10; + optional int64 field11 = 11; +} diff --git a/third_party/upb/tests/json/enum_from_separate_file.proto b/third_party/upb/tests/json/enum_from_separate_file.proto new file mode 100644 index 00000000000..ceb9b42aa1c --- /dev/null +++ b/third_party/upb/tests/json/enum_from_separate_file.proto @@ -0,0 +1,9 @@ +syntax = "proto2"; + +import "tests/json/test.proto"; + +package upb.test.json; + +message ImportEnum { + optional MyEnum e = 1; +} diff --git a/third_party/upb/tests/json/test.proto b/third_party/upb/tests/json/test.proto new file mode 100644 index 00000000000..2db0388d778 --- /dev/null +++ b/third_party/upb/tests/json/test.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package upb.test.json; + +message TestMessage { + int32 optional_int32 = 1; + int64 optional_int64 = 2; + int32 optional_uint32 = 3; + int64 optional_uint64 = 4; + string optional_string = 5; + bytes optional_bytes = 6; + bool optional_bool = 7; + SubMessage optional_msg = 8; + MyEnum optional_enum = 9; + + repeated int32 repeated_int32 = 11; + repeated int64 repeated_int64 = 12; + repeated uint32 repeated_uint32 = 13; + repeated uint64 repeated_uint64 = 14; + repeated string repeated_string = 15; + repeated bytes repeated_bytes = 16; + repeated bool repeated_bool = 17; + repeated SubMessage repeated_msg = 18; + repeated MyEnum repeated_enum = 19; + + map map_string_string = 20; + map map_int32_string = 21; + map map_bool_string = 22; + map map_string_int32 = 23; + map map_string_bool = 24; + map map_string_msg = 25; + + oneof o { + int32 oneof_int32 = 26; + int64 oneof_int64 = 27; + } +} + +message SubMessage { + int32 foo = 1; +} + +enum MyEnum { + A = 0; + B = 1; + C = 2; +} diff --git a/third_party/upb/tests/json/test.proto.pb b/third_party/upb/tests/json/test.proto.pb new file mode 100644 index 00000000000..94b8b92e616 Binary files /dev/null and b/third_party/upb/tests/json/test.proto.pb differ diff --git a/third_party/upb/tests/json/test_json.cc b/third_party/upb/tests/json/test_json.cc new file mode 100644 index 00000000000..66508179b3f --- /dev/null +++ b/third_party/upb/tests/json/test_json.cc @@ -0,0 +1,256 @@ +/* + * + * A set of tests for JSON parsing and serialization. + */ + +#include "tests/json/test.upbdefs.h" +#include "tests/json/test.upb.h" // Test that it compiles for C++. +#include "tests/test_util.h" +#include "tests/upb_test.h" +#include "upb/handlers.h" +#include "upb/json/parser.h" +#include "upb/json/printer.h" +#include "upb/upb.h" + +#include + +#include "upb/port_def.inc" + +// Macros for readability in test case list: allows us to give TEST("...") / +// EXPECT("...") pairs. +#define TEST(x) x +#define EXPECT_SAME NULL +#define EXPECT(x) x +#define TEST_SENTINEL { NULL, NULL } + +struct TestCase { + const char* input; + const char* expected; +}; + +bool verbose = false; + +static TestCase kTestRoundtripMessages[] = { + // Test most fields here. + { + TEST("{\"optionalInt32\":-42,\"optionalString\":\"Test\\u0001Message\"," + "\"optionalMsg\":{\"foo\":42}," + "\"optionalBool\":true,\"repeatedMsg\":[{\"foo\":1}," + "{\"foo\":2}]}"), + EXPECT_SAME + }, + // We must also recognize raw proto names. + { + TEST("{\"optional_int32\":-42,\"optional_string\":\"Test\\u0001Message\"," + "\"optional_msg\":{\"foo\":42}," + "\"optional_bool\":true,\"repeated_msg\":[{\"foo\":1}," + "{\"foo\":2}]}"), + EXPECT("{\"optionalInt32\":-42,\"optionalString\":\"Test\\u0001Message\"," + "\"optionalMsg\":{\"foo\":42}," + "\"optionalBool\":true,\"repeatedMsg\":[{\"foo\":1}," + "{\"foo\":2}]}") + }, + // Test special escapes in strings. + { + TEST("{\"repeatedString\":[\"\\b\",\"\\r\",\"\\n\",\"\\f\",\"\\t\"," + "\"\uFFFF\"]}"), + EXPECT_SAME + }, + // Test enum symbolic names. + { + // The common case: parse and print the symbolic name. + TEST("{\"optionalEnum\":\"A\"}"), + EXPECT_SAME + }, + { + // Unknown enum value: will be printed as an integer. + TEST("{\"optionalEnum\":42}"), + EXPECT_SAME + }, + { + // Known enum value: we're happy to parse an integer but we will re-emit the + // symbolic name. + TEST("{\"optionalEnum\":1}"), + EXPECT("{\"optionalEnum\":\"B\"}") + }, + // UTF-8 tests: escapes -> literal UTF8 in output. + { + // Note double escape on \uXXXX: we want the escape to be processed by the + // JSON parser, not by the C++ compiler! + TEST("{\"optionalString\":\"\\u007F\"}"), + EXPECT("{\"optionalString\":\"\x7F\"}") + }, + { + TEST("{\"optionalString\":\"\\u0080\"}"), + EXPECT("{\"optionalString\":\"\xC2\x80\"}") + }, + { + TEST("{\"optionalString\":\"\\u07FF\"}"), + EXPECT("{\"optionalString\":\"\xDF\xBF\"}") + }, + { + TEST("{\"optionalString\":\"\\u0800\"}"), + EXPECT("{\"optionalString\":\"\xE0\xA0\x80\"}") + }, + { + TEST("{\"optionalString\":\"\\uFFFF\"}"), + EXPECT("{\"optionalString\":\"\xEF\xBF\xBF\"}") + }, + // map-field tests + { + TEST("{\"mapStringString\":{\"a\":\"value1\",\"b\":\"value2\"," + "\"c\":\"value3\"}}"), + EXPECT_SAME + }, + { + TEST("{\"mapInt32String\":{\"1\":\"value1\",\"-1\":\"value2\"," + "\"1234\":\"value3\"}}"), + EXPECT_SAME + }, + { + TEST("{\"mapBoolString\":{\"false\":\"value1\",\"true\":\"value2\"}}"), + EXPECT_SAME + }, + { + TEST("{\"mapStringInt32\":{\"asdf\":1234,\"jkl;\":-1}}"), + EXPECT_SAME + }, + { + TEST("{\"mapStringBool\":{\"asdf\":true,\"jkl;\":false}}"), + EXPECT_SAME + }, + { + TEST("{\"mapStringMsg\":{\"asdf\":{\"foo\":42},\"jkl;\":{\"foo\":84}}}"), + EXPECT_SAME + }, + TEST_SENTINEL +}; + +static TestCase kTestRoundtripMessagesPreserve[] = { + // Test most fields here. + { + TEST("{\"optional_int32\":-42,\"optional_string\":\"Test\\u0001Message\"," + "\"optional_msg\":{\"foo\":42}," + "\"optional_bool\":true,\"repeated_msg\":[{\"foo\":1}," + "{\"foo\":2}]}"), + EXPECT_SAME + }, + TEST_SENTINEL +}; + +class StringSink { + public: + StringSink() { + upb_byteshandler_init(&byteshandler_); + upb_byteshandler_setstring(&byteshandler_, &str_handler, NULL); + upb_bytessink_reset(&bytessink_, &byteshandler_, &s_); + } + ~StringSink() { } + + upb_bytessink Sink() { return bytessink_; } + + const std::string& Data() { return s_; } + + private: + + static size_t str_handler(void* _closure, const void* hd, + const char* data, size_t len, + const upb_bufhandle* handle) { + UPB_UNUSED(hd); + UPB_UNUSED(handle); + std::string* s = static_cast(_closure); + std::string appended(data, len); + s->append(data, len); + return len; + } + + upb_byteshandler byteshandler_; + upb_bytessink bytessink_; + std::string s_; +}; + +void test_json_roundtrip_message(const char* json_src, + const char* json_expected, + const upb::Handlers* serialize_handlers, + const upb::json::ParserMethodPtr parser_method, + int seam) { + VerboseParserEnvironment env(verbose); + StringSink data_sink; + upb::json::PrinterPtr printer = upb::json::PrinterPtr::Create( + env.arena(), serialize_handlers, data_sink.Sink()); + upb::json::ParserPtr parser = upb::json::ParserPtr::Create( + env.arena(), parser_method, NULL, printer.input(), env.status(), false); + env.ResetBytesSink(parser.input()); + env.Reset(json_src, strlen(json_src), false, false); + + bool ok = env.Start() && + env.ParseBuffer(seam) && + env.ParseBuffer(-1) && + env.End(); + + ASSERT(ok); + ASSERT(env.CheckConsistency()); + + if (memcmp(json_expected, + data_sink.Data().data(), + data_sink.Data().size())) { + fprintf(stderr, + "JSON parse/serialize roundtrip result differs:\n" + "Original:\n%s\nParsed/Serialized:\n%s\n", + json_src, data_sink.Data().c_str()); + abort(); + } +} + +// Starts with a message in JSON format, parses and directly serializes again, +// and compares the result. +void test_json_roundtrip() { + upb::SymbolTable symtab; + upb::HandlerCache serialize_handlercache( + upb::json::PrinterPtr::NewCache(false)); + upb::json::CodeCache parse_codecache; + + upb::MessageDefPtr md(upb_test_json_TestMessage_getmsgdef(symtab.ptr())); + ASSERT(md); + const upb::Handlers* serialize_handlers = serialize_handlercache.Get(md); + const upb::json::ParserMethodPtr parser_method = parse_codecache.Get(md); + ASSERT(serialize_handlers); + + for (const TestCase* test_case = kTestRoundtripMessages; + test_case->input != NULL; test_case++) { + const char *expected = + (test_case->expected == EXPECT_SAME) ? + test_case->input : + test_case->expected; + + for (size_t i = 0; i < strlen(test_case->input); i++) { + test_json_roundtrip_message(test_case->input, expected, + serialize_handlers, parser_method, i); + } + } + + serialize_handlercache = upb::json::PrinterPtr::NewCache(true); + serialize_handlers = serialize_handlercache.Get(md); + + for (const TestCase* test_case = kTestRoundtripMessagesPreserve; + test_case->input != NULL; test_case++) { + const char *expected = + (test_case->expected == EXPECT_SAME) ? + test_case->input : + test_case->expected; + + for (size_t i = 0; i < strlen(test_case->input); i++) { + test_json_roundtrip_message(test_case->input, expected, + serialize_handlers, parser_method, i); + } + } +} + +extern "C" { +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + test_json_roundtrip(); + return 0; +} +} diff --git a/third_party/upb/tests/pb/test_decoder.cc b/third_party/upb/tests/pb/test_decoder.cc new file mode 100644 index 00000000000..5cc59e40c92 --- /dev/null +++ b/third_party/upb/tests/pb/test_decoder.cc @@ -0,0 +1,1203 @@ +/* + * + * An exhaustive set of tests for parsing both valid and invalid protobuf + * input, with buffer breaks in arbitrary places. + * + * Tests to add: + * - string/bytes + * - unknown field handler called appropriately + * - unknown fields can be inserted in random places + * - fuzzing of valid input + * - resource limits (max stack depth, max string len) + * - testing of groups + * - more throrough testing of sequences + * - test skipping of submessages + * - test suspending the decoder + * - buffers that are close enough to the end of the address space that + * pointers overflow (this might be difficult). + * - a few "kitchen sink" examples (one proto that uses all types, lots + * of submsg/sequences, etc. + * - test different handlers at every level and whether handlers fire at + * the correct field path. + * - test skips that extend past the end of current buffer (where decoder + * returns value greater than the size param). + */ + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS // For PRIuS, etc. +#endif + +#include +#include +#include +#include +#include +#include + +#include "tests/test_util.h" +#include "tests/upb_test.h" +#include "tests/pb/test_decoder.upbdefs.h" + +#ifdef AMALGAMATED +#include "upb.h" +#else // AMALGAMATED +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/pb/varint.int.h" +#include "upb/upb.h" +#endif // !AMALGAMATED + +#include "upb/port_def.inc" + +#undef PRINT_FAILURE +#define PRINT_FAILURE(expr) \ + fprintf(stderr, "Assertion failed: %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, "expr: %s\n", #expr); \ + if (testhash) { \ + fprintf(stderr, "assertion failed running test %x.\n", testhash); \ + if (!filter_hash) { \ + fprintf(stderr, \ + "Run with the arg %x to run only this test. " \ + "(This will also turn on extra debugging output)\n", \ + testhash); \ + } \ + fprintf(stderr, "Failed at %02.2f%% through tests.\n", \ + (float)completed * 100 / total); \ + } + +#define MAX_NESTING 64 + +#define LINE(x) x "\n" + +uint32_t filter_hash = 0; +double completed; +double total; +double *count; + +enum TestMode { + COUNT_ONLY = 1, + NO_HANDLERS = 2, + ALL_HANDLERS = 3 +} test_mode; + +// Copied from decoder.c, since this is not a public interface. +typedef struct { + uint8_t native_wire_type; + bool is_numeric; +} upb_decoder_typeinfo; + +static const upb_decoder_typeinfo upb_decoder_types[] = { + {UPB_WIRE_TYPE_END_GROUP, false}, // ENDGROUP + {UPB_WIRE_TYPE_64BIT, true}, // DOUBLE + {UPB_WIRE_TYPE_32BIT, true}, // FLOAT + {UPB_WIRE_TYPE_VARINT, true}, // INT64 + {UPB_WIRE_TYPE_VARINT, true}, // UINT64 + {UPB_WIRE_TYPE_VARINT, true}, // INT32 + {UPB_WIRE_TYPE_64BIT, true}, // FIXED64 + {UPB_WIRE_TYPE_32BIT, true}, // FIXED32 + {UPB_WIRE_TYPE_VARINT, true}, // BOOL + {UPB_WIRE_TYPE_DELIMITED, false}, // STRING + {UPB_WIRE_TYPE_START_GROUP, false}, // GROUP + {UPB_WIRE_TYPE_DELIMITED, false}, // MESSAGE + {UPB_WIRE_TYPE_DELIMITED, false}, // BYTES + {UPB_WIRE_TYPE_VARINT, true}, // UINT32 + {UPB_WIRE_TYPE_VARINT, true}, // ENUM + {UPB_WIRE_TYPE_32BIT, true}, // SFIXED32 + {UPB_WIRE_TYPE_64BIT, true}, // SFIXED64 + {UPB_WIRE_TYPE_VARINT, true}, // SINT32 + {UPB_WIRE_TYPE_VARINT, true}, // SINT64 +}; + +#ifndef USE_GOOGLE +using std::string; +#endif + +void vappendf(string* str, const char *format, va_list args) { + va_list copy; + _upb_va_copy(copy, args); + + int count = vsnprintf(NULL, 0, format, args); + if (count >= 0) + { + UPB_ASSERT(count < 32768); + char *buffer = new char[count + 1]; + UPB_ASSERT(buffer); + count = vsnprintf(buffer, count + 1, format, copy); + UPB_ASSERT(count >= 0); + str->append(buffer, count); + delete [] buffer; + } + va_end(copy); +} + +void appendf(string* str, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + vappendf(str, fmt, args); + va_end(args); +} + +void PrintBinary(const string& str) { + for (size_t i = 0; i < str.size(); i++) { + if (isprint(str[i])) { + fprintf(stderr, "%c", str[i]); + } else { + fprintf(stderr, "\\x%02x", (int)(uint8_t)str[i]); + } + } +} + +/* Routines for building arbitrary protos *************************************/ + +const string empty; + +string cat(const string& a, const string& b, + const string& c = empty, + const string& d = empty, + const string& e = empty, + const string& f = empty, + const string& g = empty, + const string& h = empty, + const string& i = empty, + const string& j = empty, + const string& k = empty, + const string& l = empty) { + string ret; + ret.reserve(a.size() + b.size() + c.size() + d.size() + e.size() + f.size() + + g.size() + h.size() + i.size() + j.size() + k.size() + l.size()); + ret.append(a); + ret.append(b); + ret.append(c); + ret.append(d); + ret.append(e); + ret.append(f); + ret.append(g); + ret.append(h); + ret.append(i); + ret.append(j); + ret.append(k); + ret.append(l); + return ret; +} + +template +string num2string(T num) { + std::ostringstream ss; + ss << num; + return ss.str(); +} + +string varint(uint64_t x) { + char buf[UPB_PB_VARINT_MAX_LEN]; + size_t len = upb_vencode64(x, buf); + return string(buf, len); +} + +// TODO: proper byte-swapping for big-endian machines. +string fixed32(void *data) { return string(static_cast(data), 4); } +string fixed64(void *data) { return string(static_cast(data), 8); } + +string delim(const string& buf) { return cat(varint(buf.size()), buf); } +string uint32(uint32_t u32) { return fixed32(&u32); } +string uint64(uint64_t u64) { return fixed64(&u64); } +string flt(float f) { return fixed32(&f); } +string dbl(double d) { return fixed64(&d); } +string zz32(int32_t x) { return varint(upb_zzenc_32(x)); } +string zz64(int64_t x) { return varint(upb_zzenc_64(x)); } + +string tag(uint32_t fieldnum, char wire_type) { + return varint((fieldnum << 3) | wire_type); +} + +string submsg(uint32_t fn, const string& buf) { + return cat( tag(fn, UPB_WIRE_TYPE_DELIMITED), delim(buf) ); +} + +string group(uint32_t fn, const string& buf) { + return cat(tag(fn, UPB_WIRE_TYPE_START_GROUP), buf, + tag(fn, UPB_WIRE_TYPE_END_GROUP)); +} + +// Like delim()/submsg(), but intentionally encodes an incorrect length. +// These help test when a delimited boundary doesn't land in the right place. +string badlen_delim(int err, const string& buf) { + return cat(varint(buf.size() + err), buf); +} + +string badlen_submsg(int err, uint32_t fn, const string& buf) { + return cat( tag(fn, UPB_WIRE_TYPE_DELIMITED), badlen_delim(err, buf) ); +} + + +/* A set of handlers that covers all .proto types *****************************/ + +// The handlers simply append to a string indicating what handlers were called. +// This string is similar to protobuf text format but fields are referred to by +// number instead of name and sequences are explicitly delimited. We indent +// using the closure depth to test that the stack of closures is properly +// handled. + +int closures[MAX_NESTING]; +string output; + +void indentbuf(string *buf, int depth) { + buf->append(2 * depth, ' '); +} + +#define NUMERIC_VALUE_HANDLER(member, ctype, fmt) \ + bool value_##member(int* depth, const uint32_t* num, ctype val) { \ + indentbuf(&output, *depth); \ + appendf(&output, "%" PRIu32 ":%" fmt "\n", *num, val); \ + return true; \ + } + +NUMERIC_VALUE_HANDLER(uint32, uint32_t, PRIu32) +NUMERIC_VALUE_HANDLER(uint64, uint64_t, PRIu64) +NUMERIC_VALUE_HANDLER(int32, int32_t, PRId32) +NUMERIC_VALUE_HANDLER(int64, int64_t, PRId64) +NUMERIC_VALUE_HANDLER(float, float, "g") +NUMERIC_VALUE_HANDLER(double, double, "g") + +bool value_bool(int* depth, const uint32_t* num, bool val) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":%s\n", *num, val ? "true" : "false"); + return true; +} + +int* startstr(int* depth, const uint32_t* num, size_t size_hint) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":(%zu)\"", *num, size_hint); + return depth + 1; +} + +size_t value_string(int* depth, const uint32_t* num, const char* buf, + size_t n, const upb_bufhandle* handle) { + UPB_UNUSED(num); + UPB_UNUSED(depth); + output.append(buf, n); + ASSERT(handle == &global_handle); + return n; +} + +bool endstr(int* depth, const uint32_t* num) { + UPB_UNUSED(num); + output.append("\n"); + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":\"\n", *num); + return true; +} + +int* startsubmsg(int* depth, const uint32_t* num) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":{\n", *num); + return depth + 1; +} + +bool endsubmsg(int* depth, const uint32_t* num) { + UPB_UNUSED(num); + indentbuf(&output, *depth); + output.append("}\n"); + return true; +} + +int* startseq(int* depth, const uint32_t* num) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":[\n", *num); + return depth + 1; +} + +bool endseq(int* depth, const uint32_t* num) { + UPB_UNUSED(num); + indentbuf(&output, *depth); + output.append("]\n"); + return true; +} + +bool startmsg(int* depth) { + indentbuf(&output, *depth); + output.append("<\n"); + return true; +} + +bool endmsg(int* depth, upb_status* status) { + UPB_UNUSED(status); + indentbuf(&output, *depth); + output.append(">\n"); + return true; +} + +void free_uint32(void *val) { + uint32_t *u32 = static_cast(val); + delete u32; +} + +template +void doreg(upb::HandlersPtr h, uint32_t num) { + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(num); + ASSERT(f); + ASSERT(h.SetValueHandler(f, UpbBind(F, new uint32_t(num)))); + if (f.IsSequence()) { + ASSERT(h.SetStartSequenceHandler(f, UpbBind(startseq, new uint32_t(num)))); + ASSERT(h.SetEndSequenceHandler(f, UpbBind(endseq, new uint32_t(num)))); + } +} + +// The repeated field number to correspond to the given non-repeated field +// number. +uint32_t rep_fn(uint32_t fn) { + return (UPB_MAX_FIELDNUMBER - 1000) + fn; +} + +#define NOP_FIELD 40 +#define UNKNOWN_FIELD 666 + +template +void reg(upb::HandlersPtr h, upb_descriptortype_t type) { + // We register both a repeated and a non-repeated field for every type. + // For the non-repeated field we make the field number the same as the + // type. For the repeated field we make it a function of the type. + doreg(h, type); + doreg(h, rep_fn(type)); +} + +void regseq(upb::HandlersPtr h, upb::FieldDefPtr f, uint32_t num) { + ASSERT(h.SetStartSequenceHandler(f, UpbBind(startseq, new uint32_t(num)))); + ASSERT(h.SetEndSequenceHandler(f, UpbBind(endseq, new uint32_t(num)))); +} + +void reg_subm(upb::HandlersPtr h, uint32_t num) { + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(num); + ASSERT(f); + if (f.IsSequence()) regseq(h, f, num); + ASSERT( + h.SetStartSubMessageHandler(f, UpbBind(startsubmsg, new uint32_t(num)))); + ASSERT(h.SetEndSubMessageHandler(f, UpbBind(endsubmsg, new uint32_t(num)))); +} + +void reg_str(upb::HandlersPtr h, uint32_t num) { + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(num); + ASSERT(f); + if (f.IsSequence()) regseq(h, f, num); + ASSERT(h.SetStartStringHandler(f, UpbBind(startstr, new uint32_t(num)))); + ASSERT(h.SetEndStringHandler(f, UpbBind(endstr, new uint32_t(num)))); + ASSERT(h.SetStringHandler(f, UpbBind(value_string, new uint32_t(num)))); +} + +struct HandlerRegisterData { + TestMode mode; +}; + +void callback(const void *closure, upb::Handlers* h_ptr) { + upb::HandlersPtr h(h_ptr); + const HandlerRegisterData* data = + static_cast(closure); + if (data->mode == ALL_HANDLERS) { + h.SetStartMessageHandler(UpbMakeHandler(startmsg)); + h.SetEndMessageHandler(UpbMakeHandler(endmsg)); + + // Register handlers for each type. + reg(h, UPB_DESCRIPTOR_TYPE_DOUBLE); + reg (h, UPB_DESCRIPTOR_TYPE_FLOAT); + reg (h, UPB_DESCRIPTOR_TYPE_INT64); + reg(h, UPB_DESCRIPTOR_TYPE_UINT64); + reg (h, UPB_DESCRIPTOR_TYPE_INT32); + reg(h, UPB_DESCRIPTOR_TYPE_FIXED64); + reg(h, UPB_DESCRIPTOR_TYPE_FIXED32); + reg (h, UPB_DESCRIPTOR_TYPE_BOOL); + reg(h, UPB_DESCRIPTOR_TYPE_UINT32); + reg (h, UPB_DESCRIPTOR_TYPE_ENUM); + reg (h, UPB_DESCRIPTOR_TYPE_SFIXED32); + reg (h, UPB_DESCRIPTOR_TYPE_SFIXED64); + reg (h, UPB_DESCRIPTOR_TYPE_SINT32); + reg (h, UPB_DESCRIPTOR_TYPE_SINT64); + + reg_str(h, UPB_DESCRIPTOR_TYPE_STRING); + reg_str(h, UPB_DESCRIPTOR_TYPE_BYTES); + reg_str(h, rep_fn(UPB_DESCRIPTOR_TYPE_STRING)); + reg_str(h, rep_fn(UPB_DESCRIPTOR_TYPE_BYTES)); + + // Register submessage/group handlers that are self-recursive + // to this type, eg: message M { optional M m = 1; } + reg_subm(h, UPB_DESCRIPTOR_TYPE_MESSAGE); + reg_subm(h, rep_fn(UPB_DESCRIPTOR_TYPE_MESSAGE)); + + if (h.message_def().full_name() == std::string("DecoderTest")) { + reg_subm(h, UPB_DESCRIPTOR_TYPE_GROUP); + reg_subm(h, rep_fn(UPB_DESCRIPTOR_TYPE_GROUP)); + } + + // For NOP_FIELD we register no handlers, so we can pad a proto freely without + // changing the output. + } +} + +/* Running of test cases ******************************************************/ + +const upb::Handlers *global_handlers; +upb::pb::DecoderMethodPtr global_method; + +upb::pb::DecoderPtr CreateDecoder(upb::Arena* arena, + upb::pb::DecoderMethodPtr method, + upb::Sink sink, upb::Status* status) { + upb::pb::DecoderPtr ret = + upb::pb::DecoderPtr::Create(arena, method, sink, status); + ret.set_max_nesting(MAX_NESTING); + return ret; +} + +uint32_t Hash(const string& proto, const string* expected_output, size_t seam1, + size_t seam2, bool may_skip) { + uint32_t hash = upb_murmur_hash2(proto.c_str(), proto.size(), 0); + if (expected_output) + hash = upb_murmur_hash2(expected_output->c_str(), expected_output->size(), hash); + hash = upb_murmur_hash2(&seam1, sizeof(seam1), hash); + hash = upb_murmur_hash2(&seam2, sizeof(seam2), hash); + hash = upb_murmur_hash2(&may_skip, sizeof(may_skip), hash); + return hash; +} + +void CheckBytesParsed(upb::pb::DecoderPtr decoder, size_t ofs) { + // We can't have parsed more data than the decoder callback is telling us it + // parsed. + ASSERT(decoder.BytesParsed() <= ofs); + + // The difference between what we've decoded and what the decoder has accepted + // represents the internally buffered amount. This amount should not exceed + // this value which comes from decoder.int.h. + ASSERT(ofs <= (decoder.BytesParsed() + UPB_DECODER_MAX_RESIDUAL_BYTES)); +} + +static bool parse(VerboseParserEnvironment* env, + upb::pb::DecoderPtr decoder, int bytes) { + CheckBytesParsed(decoder, env->ofs()); + bool ret = env->ParseBuffer(bytes); + if (ret) { + CheckBytesParsed(decoder, env->ofs()); + } + + return ret; +} + +void do_run_decoder(VerboseParserEnvironment* env, upb::pb::DecoderPtr decoder, + const string& proto, const string* expected_output, + size_t i, size_t j, bool may_skip) { + env->Reset(proto.c_str(), proto.size(), may_skip, expected_output == NULL); + decoder.Reset(); + + testhash = Hash(proto, expected_output, i, j, may_skip); + if (filter_hash && testhash != filter_hash) return; + if (test_mode != COUNT_ONLY) { + output.clear(); + + if (filter_hash) { + fprintf(stderr, "RUNNING TEST CASE, hash=%x\n", testhash); + fprintf(stderr, "Input (len=%u): ", (unsigned)proto.size()); + PrintBinary(proto); + fprintf(stderr, "\n"); + if (expected_output) { + if (test_mode == ALL_HANDLERS) { + fprintf(stderr, "Expected output: %s\n", expected_output->c_str()); + } else if (test_mode == NO_HANDLERS) { + fprintf(stderr, + "No handlers are registered, BUT if they were " + "the expected output would be: %s\n", + expected_output->c_str()); + } + } else { + fprintf(stderr, "Expected to FAIL\n"); + } + } + + bool ok = env->Start() && + parse(env, decoder, i) && + parse(env, decoder, j - i) && + parse(env, decoder, -1) && + env->End(); + + ASSERT(env->CheckConsistency()); + + if (test_mode == ALL_HANDLERS) { + if (expected_output) { + if (output != *expected_output) { + fprintf(stderr, "Text mismatch: '%s' vs '%s'\n", + output.c_str(), expected_output->c_str()); + } + ASSERT(ok); + ASSERT(output == *expected_output); + } else { + if (ok) { + fprintf(stderr, "Didn't expect ok result, but got output: '%s'\n", + output.c_str()); + } + ASSERT(!ok); + } + } + } + (*count)++; +} + +void run_decoder(const string& proto, const string* expected_output) { + VerboseParserEnvironment env(filter_hash != 0); + upb::Sink sink(global_handlers, &closures[0]); + upb::pb::DecoderPtr decoder = CreateDecoder(env.arena(), global_method, sink, env.status()); + env.ResetBytesSink(decoder.input()); + for (size_t i = 0; i < proto.size(); i++) { + for (size_t j = i; j < UPB_MIN(proto.size(), i + 5); j++) { + do_run_decoder(&env, decoder, proto, expected_output, i, j, true); + if (env.SkippedWithNull()) { + do_run_decoder(&env, decoder, proto, expected_output, i, j, false); + } + } + } + testhash = 0; +} + +const static string thirty_byte_nop = cat( + tag(NOP_FIELD, UPB_WIRE_TYPE_DELIMITED), delim(string(30, 'X')) ); + +// Indents and wraps text as if it were a submessage with this field number +string wrap_text(int32_t fn, const string& text) { + string wrapped_text = text; + size_t pos = 0; + string replace_with = "\n "; + while ((pos = wrapped_text.find("\n", pos)) != string::npos && + pos != wrapped_text.size() - 1) { + wrapped_text.replace(pos, 1, replace_with); + pos += replace_with.size(); + } + wrapped_text = cat( + LINE("<"), + num2string(fn), LINE(":{") + " ", wrapped_text, + LINE("}") + LINE(">")); + return wrapped_text; +} + +void assert_successful_parse(const string& proto, + const char *expected_fmt, ...) { + string expected_text; + va_list args; + va_start(args, expected_fmt); + vappendf(&expected_text, expected_fmt, args); + va_end(args); + // To test both middle-of-buffer and end-of-buffer code paths, + // repeat once with no-op padding data at the end of buffer. + run_decoder(proto, &expected_text); + run_decoder(cat( proto, thirty_byte_nop ), &expected_text); + + // Test that this also works when wrapped in a submessage or group. + // Indent the expected text one level and wrap it. + string wrapped_text1 = wrap_text(UPB_DESCRIPTOR_TYPE_MESSAGE, expected_text); + string wrapped_text2 = wrap_text(UPB_DESCRIPTOR_TYPE_GROUP, expected_text); + + run_decoder(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, proto), &wrapped_text1); + run_decoder(group(UPB_DESCRIPTOR_TYPE_GROUP, proto), &wrapped_text2); +} + +void assert_does_not_parse_at_eof(const string& proto) { + run_decoder(proto, NULL); + + // Also test that we fail to parse at end-of-submessage, not just + // end-of-message. But skip this if we have no handlers, because in that + // case we won't descend into the submessage. + if (test_mode != NO_HANDLERS) { + run_decoder(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, proto), NULL); + run_decoder(cat(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, proto), + thirty_byte_nop), NULL); + } +} + +void assert_does_not_parse(const string& proto) { + // Test that the error is caught both at end-of-buffer and middle-of-buffer. + assert_does_not_parse_at_eof(proto); + assert_does_not_parse_at_eof(cat( proto, thirty_byte_nop )); +} + + +/* The actual tests ***********************************************************/ + +void test_premature_eof_for_type(upb_descriptortype_t type) { + // Incomplete values for each wire type. + static const string incompletes[6] = { + string("\x80"), // UPB_WIRE_TYPE_VARINT + string("abcdefg"), // UPB_WIRE_TYPE_64BIT + string("\x80"), // UPB_WIRE_TYPE_DELIMITED (partial length) + string(), // UPB_WIRE_TYPE_START_GROUP (no value required) + string(), // UPB_WIRE_TYPE_END_GROUP (no value required) + string("abc") // UPB_WIRE_TYPE_32BIT + }; + + uint32_t fieldnum = type; + uint32_t rep_fieldnum = rep_fn(type); + int wire_type = upb_decoder_types[type].native_wire_type; + const string& incomplete = incompletes[wire_type]; + + // EOF before a known non-repeated value. + assert_does_not_parse_at_eof(tag(fieldnum, wire_type)); + + // EOF before a known repeated value. + assert_does_not_parse_at_eof(tag(rep_fieldnum, wire_type)); + + // EOF before an unknown value. + assert_does_not_parse_at_eof(tag(UNKNOWN_FIELD, wire_type)); + + // EOF inside a known non-repeated value. + assert_does_not_parse_at_eof( + cat( tag(fieldnum, wire_type), incomplete )); + + // EOF inside a known repeated value. + assert_does_not_parse_at_eof( + cat( tag(rep_fieldnum, wire_type), incomplete )); + + // EOF inside an unknown value. + assert_does_not_parse_at_eof( + cat( tag(UNKNOWN_FIELD, wire_type), incomplete )); + + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + // EOF in the middle of delimited data for known non-repeated value. + assert_does_not_parse_at_eof( + cat( tag(fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for known repeated value. + assert_does_not_parse_at_eof( + cat( tag(rep_fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for unknown value. + assert_does_not_parse_at_eof( + cat( tag(UNKNOWN_FIELD, wire_type), varint(1) )); + + if (type == UPB_DESCRIPTOR_TYPE_MESSAGE) { + // Submessage ends in the middle of a value. + string incomplete_submsg = + cat ( tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), + incompletes[UPB_WIRE_TYPE_VARINT] ); + assert_does_not_parse( + cat( tag(fieldnum, UPB_WIRE_TYPE_DELIMITED), + varint(incomplete_submsg.size()), + incomplete_submsg )); + } + } else { + // Packed region ends in the middle of a value. + assert_does_not_parse( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), + varint(incomplete.size()), + incomplete )); + + // EOF in the middle of packed region. + assert_does_not_parse_at_eof( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), varint(1) )); + } +} + +// "33" and "66" are just two random values that all numeric types can +// represent. +void test_valid_data_for_type(upb_descriptortype_t type, + const string& enc33, const string& enc66) { + uint32_t fieldnum = type; + uint32_t rep_fieldnum = rep_fn(type); + int wire_type = upb_decoder_types[type].native_wire_type; + + // Non-repeated + assert_successful_parse( + cat( tag(fieldnum, wire_type), enc33, + tag(fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:33") + LINE("%u:66") + LINE(">"), fieldnum, fieldnum); + + // Non-packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, wire_type), enc33, + tag(rep_fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); + + // Packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), + delim(cat( enc33, enc66 )) ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); +} + +void test_valid_data_for_signed_type(upb_descriptortype_t type, + const string& enc33, const string& enc66) { + uint32_t fieldnum = type; + uint32_t rep_fieldnum = rep_fn(type); + int wire_type = upb_decoder_types[type].native_wire_type; + + // Non-repeated + assert_successful_parse( + cat( tag(fieldnum, wire_type), enc33, + tag(fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:33") + LINE("%u:-66") + LINE(">"), fieldnum, fieldnum); + + // Non-packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, wire_type), enc33, + tag(rep_fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:-66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); + + // Packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), + delim(cat( enc33, enc66 )) ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:-66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); +} + +// Test that invalid protobufs are properly detected (without crashing) and +// have an error reported. Field numbers match registered handlers above. +void test_invalid() { + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_DOUBLE); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_FLOAT); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_INT64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_UINT64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_INT32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_FIXED64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_FIXED32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_BOOL); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_STRING); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_BYTES); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_UINT32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_ENUM); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SFIXED32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SFIXED64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SINT32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SINT64); + + // EOF inside a tag's varint. + assert_does_not_parse_at_eof( string("\x80") ); + + // EOF inside a known group. + // TODO(haberman): add group to decoder test schema. + //assert_does_not_parse_at_eof( tag(4, UPB_WIRE_TYPE_START_GROUP) ); + + // EOF inside an unknown group. + assert_does_not_parse_at_eof( tag(UNKNOWN_FIELD, UPB_WIRE_TYPE_START_GROUP) ); + + // End group that we are not currently in. + assert_does_not_parse( tag(4, UPB_WIRE_TYPE_END_GROUP) ); + + // Field number is 0. + assert_does_not_parse( + cat( tag(0, UPB_WIRE_TYPE_DELIMITED), varint(0) )); + // The previous test alone did not catch this particular pattern which could + // corrupt the internal state. + assert_does_not_parse( + cat( tag(0, UPB_WIRE_TYPE_64BIT), uint64(0) )); + + // Field number is too large. + assert_does_not_parse( + cat( tag(UPB_MAX_FIELDNUMBER + 1, UPB_WIRE_TYPE_DELIMITED), + varint(0) )); + + // Known group inside a submessage has ENDGROUP tag AFTER submessage end. + assert_does_not_parse( + cat ( submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, + tag(UPB_DESCRIPTOR_TYPE_GROUP, UPB_WIRE_TYPE_START_GROUP)), + tag(UPB_DESCRIPTOR_TYPE_GROUP, UPB_WIRE_TYPE_END_GROUP))); + + // Unknown string extends past enclosing submessage. + assert_does_not_parse( + cat (badlen_submsg(-1, UPB_DESCRIPTOR_TYPE_MESSAGE, + submsg(12345, string(" "))), + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, string(" ")))); + + // Unknown fixed-length field extends past enclosing submessage. + assert_does_not_parse( + cat (badlen_submsg(-1, UPB_DESCRIPTOR_TYPE_MESSAGE, + cat( tag(12345, UPB_WIRE_TYPE_64BIT), uint64(0))), + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, string(" ")))); + + // Test exceeding the resource limit of stack depth. + if (test_mode != NO_HANDLERS) { + string buf; + for (int i = 0; i <= MAX_NESTING; i++) { + buf.assign(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, buf)); + } + assert_does_not_parse(buf); + } +} + +void test_valid() { + // Empty protobuf. + assert_successful_parse(string(""), "<\n>\n"); + + // Empty protobuf where we never call PutString between + // StartString/EndString. + + // Randomly generated hash for this test, hope it doesn't conflict with others + // by chance. + const uint32_t emptyhash = 0x5709be8e; + if (!filter_hash || filter_hash == testhash) { + testhash = emptyhash; + upb::Status status; + upb::Arena arena; + upb::Sink sink(global_handlers, &closures[0]); + upb::pb::DecoderPtr decoder = + CreateDecoder(&arena, global_method, sink, &status); + output.clear(); + bool ok = upb::PutBuffer(std::string(), decoder.input()); + ASSERT(ok); + ASSERT(status.ok()); + if (test_mode == ALL_HANDLERS) { + ASSERT(output == string("<\n>\n")); + } + } + + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_DOUBLE, + dbl(33), + dbl(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_FLOAT, flt(33), flt(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_INT64, + varint(33), + varint(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_INT32, + varint(33), + varint(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_ENUM, + varint(33), + varint(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SFIXED32, + uint32(33), + uint32(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SFIXED64, + uint64(33), + uint64(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SINT32, + zz32(33), + zz32(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SINT64, + zz64(33), + zz64(-66)); + + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_UINT64, varint(33), varint(66)); + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_UINT32, varint(33), varint(66)); + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_FIXED64, uint64(33), uint64(66)); + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_FIXED32, uint32(33), uint32(66)); + + // Unknown fields. + int int32_type = UPB_DESCRIPTOR_TYPE_INT32; + int msg_type = UPB_DESCRIPTOR_TYPE_MESSAGE; + assert_successful_parse( + cat( tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678) ), + "<\n>\n"); + assert_successful_parse( + cat( tag(12345, UPB_WIRE_TYPE_32BIT), uint32(2345678) ), + "<\n>\n"); + assert_successful_parse( + cat( tag(12345, UPB_WIRE_TYPE_64BIT), uint64(2345678) ), + "<\n>\n"); + assert_successful_parse( + submsg(12345, string(" ")), + "<\n>\n"); + + // Unknown field inside a known submessage. + assert_successful_parse( + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, submsg(12345, string(" "))), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" >") + LINE("}") + LINE(">"), UPB_DESCRIPTOR_TYPE_MESSAGE); + + assert_successful_parse( + cat (submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, submsg(12345, string(" "))), + tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), + varint(5)), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" >") + LINE("}") + LINE("%u:5") + LINE(">"), UPB_DESCRIPTOR_TYPE_MESSAGE, UPB_DESCRIPTOR_TYPE_INT32); + + // This triggered a previous bug in the decoder. + assert_successful_parse( + cat( tag(UPB_DESCRIPTOR_TYPE_SFIXED32, UPB_WIRE_TYPE_VARINT), + varint(0) ), + "<\n>\n"); + + assert_successful_parse( + cat( + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, + cat( tag(int32_type, UPB_WIRE_TYPE_VARINT), varint(2345678), + tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678) ))), + tag(int32_type, UPB_WIRE_TYPE_VARINT), varint(22222)), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" %u:{") + LINE(" <") + LINE(" %u:2345678") + LINE(" >") + LINE(" }") + LINE(" >") + LINE("}") + LINE("%u:22222") + LINE(">"), msg_type, msg_type, int32_type, int32_type); + + assert_successful_parse( + cat( tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), varint(1), + tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678) ), + LINE("<") + LINE("%u:1") + LINE(">"), UPB_DESCRIPTOR_TYPE_INT32); + + // String inside submsg. + uint32_t msg_fn = UPB_DESCRIPTOR_TYPE_MESSAGE; + assert_successful_parse( + submsg(msg_fn, + cat ( tag(UPB_DESCRIPTOR_TYPE_STRING, UPB_WIRE_TYPE_DELIMITED), + delim(string("abcde")) + ) + ), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" %u:(5)\"abcde") + LINE(" %u:\"") + LINE(" >") + LINE("}") + LINE(">"), msg_fn, UPB_DESCRIPTOR_TYPE_STRING, + UPB_DESCRIPTOR_TYPE_STRING); + + // Test implicit startseq/endseq. + uint32_t repfl_fn = rep_fn(UPB_DESCRIPTOR_TYPE_FLOAT); + uint32_t repdb_fn = rep_fn(UPB_DESCRIPTOR_TYPE_DOUBLE); + assert_successful_parse( + cat( tag(repfl_fn, UPB_WIRE_TYPE_32BIT), flt(33), + tag(repdb_fn, UPB_WIRE_TYPE_64BIT), dbl(66) ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE("]") + LINE("%u:[") + LINE(" %u:66") + LINE("]") + LINE(">"), repfl_fn, repfl_fn, repdb_fn, repdb_fn); + + // Submessage tests. + assert_successful_parse( + submsg(msg_fn, submsg(msg_fn, submsg(msg_fn, string()))), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" %u:{") + LINE(" <") + LINE(" %u:{") + LINE(" <") + LINE(" >") + LINE(" }") + LINE(" >") + LINE(" }") + LINE(" >") + LINE("}") + LINE(">"), msg_fn, msg_fn, msg_fn); + + uint32_t repm_fn = rep_fn(UPB_DESCRIPTOR_TYPE_MESSAGE); + assert_successful_parse( + submsg(repm_fn, submsg(repm_fn, string())), + LINE("<") + LINE("%u:[") + LINE(" %u:{") + LINE(" <") + LINE(" %u:[") + LINE(" %u:{") + LINE(" <") + LINE(" >") + LINE(" }") + LINE(" ]") + LINE(" >") + LINE(" }") + LINE("]") + LINE(">"), repm_fn, repm_fn, repm_fn, repm_fn); + + // Test unknown group. + uint32_t unknown_group_fn = 12321; + assert_successful_parse( + cat( tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) ), + LINE("<") + LINE(">") + ); + + // Test some unknown fields inside an unknown group. + const string unknown_group_with_data = + cat( + tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678), + tag(123456789, UPB_WIRE_TYPE_32BIT), uint32(2345678), + tag(123477, UPB_WIRE_TYPE_64BIT), uint64(2345678), + tag(123, UPB_WIRE_TYPE_DELIMITED), varint(0), + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) + ); + + // Nested unknown group with data. + assert_successful_parse( + cat( + tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + unknown_group_with_data, + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP), + tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), varint(1) + ), + LINE("<") + LINE("%u:1") + LINE(">"), + UPB_DESCRIPTOR_TYPE_INT32 + ); + + assert_successful_parse( + cat( tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + tag(unknown_group_fn + 1, UPB_WIRE_TYPE_START_GROUP), + tag(unknown_group_fn + 1, UPB_WIRE_TYPE_END_GROUP), + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) ), + LINE("<") + LINE(">") + ); + + // Staying within the stack limit should work properly. + string buf; + string textbuf; + int total = MAX_NESTING - 1; + for (int i = 0; i < total; i++) { + buf.assign(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, buf)); + indentbuf(&textbuf, i); + textbuf.append("<\n"); + indentbuf(&textbuf, i); + appendf(&textbuf, "%u:{\n", UPB_DESCRIPTOR_TYPE_MESSAGE); + } + indentbuf(&textbuf, total); + textbuf.append("<\n"); + indentbuf(&textbuf, total); + textbuf.append(">\n"); + for (int i = 0; i < total; i++) { + indentbuf(&textbuf, total - i - 1); + textbuf.append("}\n"); + indentbuf(&textbuf, total - i - 1); + textbuf.append(">\n"); + } + // Have to use run_decoder directly, because we are at max nesting and can't + // afford the extra nesting that assert_successful_parse() will do. + run_decoder(buf, &textbuf); +} + +void empty_callback(const void *closure, upb::Handlers* h_ptr) {} + +void test_emptyhandlers(upb::SymbolTable* symtab) { + // Create an empty handlers to make sure that the decoder can handle empty + // messages. + HandlerRegisterData handlerdata; + handlerdata.mode = test_mode; + + upb::HandlerCache handler_cache(empty_callback, &handlerdata); + upb::pb::CodeCache pb_code_cache(&handler_cache); + + upb::MessageDefPtr md = upb::MessageDefPtr(Empty_getmsgdef(symtab->ptr())); + global_handlers = handler_cache.Get(md); + global_method = pb_code_cache.Get(md); + + // TODO: also test the case where a message has fields, but the fields are + // submessage fields and have no handlers. This also results in a decoder + // method with no field-handling code. + + // Ensure that the method can run with empty and non-empty input. + string test_unknown_field_msg = + cat(tag(1, UPB_WIRE_TYPE_VARINT), varint(42), + tag(2, UPB_WIRE_TYPE_DELIMITED), delim("My test data")); + const struct { + const char* data; + size_t length; + } testdata[] = { + { "", 0 }, + { test_unknown_field_msg.data(), test_unknown_field_msg.size() }, + { NULL, 0 }, + }; + for (int i = 0; testdata[i].data; i++) { + VerboseParserEnvironment env(filter_hash != 0); + upb::Sink sink(global_method.dest_handlers(), &closures[0]); + upb::pb::DecoderPtr decoder = + CreateDecoder(env.arena(), global_method, sink, env.status()); + env.ResetBytesSink(decoder.input()); + env.Reset(testdata[i].data, testdata[i].length, true, false); + ASSERT(env.Start()); + ASSERT(env.ParseBuffer(-1)); + ASSERT(env.End()); + ASSERT(env.CheckConsistency()); + } +} + +void run_tests() { + HandlerRegisterData handlerdata; + handlerdata.mode = test_mode; + + upb::SymbolTable symtab; + upb::HandlerCache handler_cache(callback, &handlerdata); + upb::pb::CodeCache pb_code_cache(&handler_cache); + + upb::MessageDefPtr md(DecoderTest_getmsgdef(symtab.ptr())); + global_handlers = handler_cache.Get(md); + global_method = pb_code_cache.Get(md); + completed = 0; + + test_invalid(); + test_valid(); + + test_emptyhandlers(&symtab); +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + if (argc > 1) + filter_hash = strtol(argv[1], NULL, 16); + for (int i = 0; i < MAX_NESTING; i++) { + closures[i] = i; + } + + // Count tests. + count = &total; + total = 0; + test_mode = COUNT_ONLY; + run_tests(); + count = &completed; + + total *= 2; // NO_HANDLERS, ALL_HANDLERS. + + test_mode = NO_HANDLERS; + run_tests(); + + test_mode = ALL_HANDLERS; + run_tests(); + + printf("All tests passed, %d assertions.\n", num_assertions); + return 0; +} + +} diff --git a/third_party/upb/tests/pb/test_decoder.proto b/third_party/upb/tests/pb/test_decoder.proto new file mode 100644 index 00000000000..e9fa6ad32ce --- /dev/null +++ b/third_party/upb/tests/pb/test_decoder.proto @@ -0,0 +1,128 @@ + +syntax = "proto2"; + +enum TestEnum { + FOO = 1; +} + +message Empty {} + +message DecoderTest { + optional double f_double = 1; + optional float f_float = 2; + optional int64 f_int64 = 3; + optional uint64 f_uint64 = 4; + optional int32 f_int32 = 5; + optional fixed64 f_fixed64 = 6; + optional fixed32 f_fixed32 = 7; + optional bool f_bool = 8; + optional string f_string = 9; + optional DecoderTest f_message = 11; + optional bytes f_bytes = 12; + optional uint32 f_uint32 = 13; + optional TestEnum f_enum = 14; + optional sfixed32 f_sfixed32 = 15; + optional sfixed64 f_sfixed64 = 16; + optional sint32 f_sint32 = 17; + optional sint64 f_sint64 = 18; + + optional string nop_field = 40; + + repeated double r_double = 536869912; + repeated float r_float = 536869913; + repeated int64 r_int64 = 536869914; + repeated uint64 r_uint64 = 536869915; + repeated int32 r_int32 = 536869916; + repeated fixed64 r_fixed64 = 536869917; + repeated fixed32 r_fixed32 = 536869918; + repeated bool r_bool = 536869919; + repeated string r_string = 536869920; + repeated DecoderTest r_message = 536869922; + repeated bytes r_bytes = 536869923; + repeated uint32 r_uint32 = 536869924; + repeated TestEnum r_enum = 536869925; + repeated sfixed32 r_sfixed32 = 536869926; + repeated sfixed64 r_sfixed64 = 536869927; + repeated sint32 r_sint32 = 536869928; + repeated sint64 r_sint64 = 536869929; + + optional group F_group = 10 { + optional double f_double = 1; + optional float f_float = 2; + optional int64 f_int64 = 3; + optional uint64 f_uint64 = 4; + optional int32 f_int32 = 5; + optional fixed64 f_fixed64 = 6; + optional fixed32 f_fixed32 = 7; + optional bool f_bool = 8; + optional string f_string = 9; + optional DecoderTest f_message = 11; + optional bytes f_bytes = 12; + optional uint32 f_uint32 = 13; + optional TestEnum f_enum = 14; + optional sfixed32 f_sfixed32 = 15; + optional sfixed64 f_sfixed64 = 16; + optional sint32 f_sint32 = 17; + optional sint64 f_sint64 = 18; + + optional string nop_field = 40; + + repeated double r_double = 536869912; + repeated float r_float = 536869913; + repeated int64 r_int64 = 536869914; + repeated uint64 r_uint64 = 536869915; + repeated int32 r_int32 = 536869916; + repeated fixed64 r_fixed64 = 536869917; + repeated fixed32 r_fixed32 = 536869918; + repeated bool r_bool = 536869919; + repeated string r_string = 536869920; + repeated DecoderTest r_message = 536869922; + repeated bytes r_bytes = 536869923; + repeated uint32 r_uint32 = 536869924; + repeated TestEnum r_enum = 536869925; + repeated sfixed32 r_sfixed32 = 536869926; + repeated sfixed64 r_sfixed64 = 536869927; + repeated sint32 r_sint32 = 536869928; + repeated sint64 r_sint64 = 536869929; + } + + optional group R_group = 536869921 { + optional double f_double = 1; + optional float f_float = 2; + optional int64 f_int64 = 3; + optional uint64 f_uint64 = 4; + optional int32 f_int32 = 5; + optional fixed64 f_fixed64 = 6; + optional fixed32 f_fixed32 = 7; + optional bool f_bool = 8; + optional string f_string = 9; + optional DecoderTest f_message = 11; + optional bytes f_bytes = 12; + optional uint32 f_uint32 = 13; + optional TestEnum f_enum = 14; + optional sfixed32 f_sfixed32 = 15; + optional sfixed64 f_sfixed64 = 16; + optional sint32 f_sint32 = 17; + optional sint64 f_sint64 = 18; + + optional string nop_field = 40; + + repeated double r_double = 536869912; + repeated float r_float = 536869913; + repeated int64 r_int64 = 536869914; + repeated uint64 r_uint64 = 536869915; + repeated int32 r_int32 = 536869916; + repeated fixed64 r_fixed64 = 536869917; + repeated fixed32 r_fixed32 = 536869918; + repeated bool r_bool = 536869919; + repeated string r_string = 536869920; + repeated DecoderTest r_message = 536869922; + repeated bytes r_bytes = 536869923; + repeated uint32 r_uint32 = 536869924; + repeated TestEnum r_enum = 536869925; + repeated sfixed32 r_sfixed32 = 536869926; + repeated sfixed64 r_sfixed64 = 536869927; + repeated sint32 r_sint32 = 536869928; + repeated sint64 r_sint64 = 536869929; + } +} diff --git a/third_party/upb/tests/pb/test_encoder.cc b/third_party/upb/tests/pb/test_encoder.cc new file mode 100644 index 00000000000..c6544beea57 --- /dev/null +++ b/third_party/upb/tests/pb/test_encoder.cc @@ -0,0 +1,48 @@ + +#include "tests/test_util.h" +#include "tests/upb_test.h" +#include "upb/bindings/stdc++/string.h" +#include "google/protobuf/descriptor.upb.h" +#include "google/protobuf/descriptor.upbdefs.h" +#include "upb/pb/decoder.h" +#include "upb/pb/encoder.h" + +#include "upb/port_def.inc" +#include + +void test_pb_roundtrip() { + std::string input( + google_protobuf_descriptor_proto_upbdefinit.descriptor.data, + google_protobuf_descriptor_proto_upbdefinit.descriptor.size); + std::cout << input.size() << "\n"; + upb::SymbolTable symtab; + upb::HandlerCache encoder_cache(upb::pb::EncoderPtr::NewCache()); + upb::pb::CodeCache decoder_cache(&encoder_cache); + upb::Arena arena; + upb::Status status; + upb::MessageDefPtr md( + google_protobuf_FileDescriptorProto_getmsgdef(symtab.ptr())); + ASSERT(md); + const upb::Handlers *encoder_handlers = encoder_cache.Get(md); + ASSERT(encoder_handlers); + const upb::pb::DecoderMethodPtr method = decoder_cache.Get(md); + + std::string output; + upb::StringSink string_sink(&output); + upb::pb::EncoderPtr encoder = + upb::pb::EncoderPtr::Create(&arena, encoder_handlers, string_sink.input()); + upb::pb::DecoderPtr decoder = + upb::pb::DecoderPtr::Create(&arena, method, encoder.input(), &status); + bool ok = upb::PutBuffer(input, decoder.input()); + ASSERT(ok); + ASSERT(input == output); +} + +extern "C" { +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + test_pb_roundtrip(); + return 0; +} +} diff --git a/third_party/upb/tests/pb/test_varint.c b/third_party/upb/tests/pb/test_varint.c new file mode 100644 index 00000000000..95a04ab9b8d --- /dev/null +++ b/third_party/upb/tests/pb/test_varint.c @@ -0,0 +1,117 @@ + +#include +#include "upb/pb/varint.int.h" +#include "tests/upb_test.h" + +#include "upb/port_def.inc" + +/* Test that we can round-trip from int->varint->int. */ +static void test_varint_for_num(upb_decoderet (*decoder)(const char*), + uint64_t num) { + char buf[16]; + size_t bytes; + upb_decoderet r; + + memset(buf, 0xff, sizeof(buf)); + bytes = upb_vencode64(num, buf); + + if (num <= UINT32_MAX) { + uint64_t encoded = upb_vencode32(num); + char buf2[16]; + upb_decoderet r; + + memset(buf2, 0, sizeof(buf2)); + memcpy(&buf2, &encoded, 8); +#ifdef UPB_BIG_ENDIAN + char swap[8]; + swap[0] = buf2[7]; + swap[1] = buf2[6]; + swap[2] = buf2[5]; + swap[3] = buf2[4]; + swap[4] = buf2[3]; + swap[5] = buf2[2]; + swap[6] = buf2[1]; + swap[7] = buf2[0]; + buf2[0] = swap[0]; + buf2[1] = swap[1]; + buf2[2] = swap[2]; + buf2[3] = swap[3]; + buf2[4] = swap[4]; + buf2[5] = swap[5]; + buf2[6] = swap[6]; + buf2[7] = swap[7]; +#endif + r = decoder(buf2); + ASSERT(r.val == num); + ASSERT(r.p == buf2 + upb_value_size(encoded)); + ASSERT(upb_zzenc_32(upb_zzdec_32(num)) == num); + } + + r = decoder(buf); + ASSERT(r.val == num); + ASSERT(r.p == buf + bytes); + ASSERT(upb_zzenc_64(upb_zzdec_64(num)) == num); +} + +static void test_varint_decoder(upb_decoderet (*decoder)(const char*)) { +#define TEST(bytes, expected_val) {\ + size_t n = sizeof(bytes) - 1; /* for NULL */ \ + char buf[UPB_PB_VARINT_MAX_LEN]; \ + upb_decoderet r; \ + memset(buf, 0xff, sizeof(buf)); \ + memcpy(buf, bytes, n); \ + r = decoder(buf); \ + ASSERT(r.val == expected_val); \ + ASSERT(r.p == buf + n); \ + } + + uint64_t num; + + char twelvebyte[16] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1}; + const char *twelvebyte_buf = twelvebyte; + /* A varint that terminates before hitting the end of the provided buffer, + * but in too many bytes (11 instead of 10). */ + upb_decoderet r = decoder(twelvebyte_buf); + ASSERT(r.p == NULL); + + TEST("\x00", 0ULL); + TEST("\x01", 1ULL); + TEST("\x81\x14", 0xa01ULL); + TEST("\x81\x03", 0x181ULL); + TEST("\x81\x83\x07", 0x1c181ULL); + TEST("\x81\x83\x87\x0f", 0x1e1c181ULL); + TEST("\x81\x83\x87\x8f\x1f", 0x1f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\x3f", 0x1f9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\x7f", 0x1fdf9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x01", 0x3fdf9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x81\x03", 0x303fdf9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x81\x83\x07", 0x8303fdf9f1e1c181ULL); +#undef TEST + + for (num = 5; num * 1.5 < UINT64_MAX; num *= 1.5) { + test_varint_for_num(decoder, num); + } + test_varint_for_num(decoder, 0); +} + + +#define TEST_VARINT_DECODER(decoder) \ + /* Create non-inline versions for convenient inspection of assembly language \ + * output. */ \ + upb_decoderet _upb_vdecode_ ## decoder(const char *p) { \ + return upb_vdecode_ ## decoder(p); \ + } \ + void test_ ## decoder(void) { \ + test_varint_decoder(&_upb_vdecode_ ## decoder); \ + } \ + +TEST_VARINT_DECODER(check2_branch32) +TEST_VARINT_DECODER(check2_branch64) + +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + test_check2_branch32(); + test_check2_branch64(); + return 0; +} diff --git a/third_party/upb/tests/test.proto b/third_party/upb/tests/test.proto new file mode 100644 index 00000000000..e790cbf06d7 --- /dev/null +++ b/third_party/upb/tests/test.proto @@ -0,0 +1,68 @@ + +// A series of messages with various kinds of cycles in them. +// +-+---+ +---+ +---+ +// V | | V | V | +// A -> B-+-> C -> D---+--->E---+ +// ^ |`---|--------^ +// +----------+----+ F + +syntax = "proto2"; + +message A { + optional B b = 1; +} + +message B { + optional B b = 1; + optional C c = 2; +} + +message C { + optional A a = 1; + optional B b = 2; + optional D d = 3; + optional E e = 4; +} + +message D { + optional A a = 1; + optional D d = 2; + optional E e = 3; +} + +message E { + optional E e = 1; +} + +message F { + optional E e = 1; +} + +// A proto with a bunch of simple primitives. +message SimplePrimitives { + optional fixed64 u64 = 1; + optional fixed32 u32 = 2; + optional double dbl = 3; + optional float flt = 5; + optional sint64 i64 = 6; + optional sint32 i32 = 7; + optional bool b = 8; + optional string str = 9; + + oneof foo { + int32 oneof_int32 = 10; + string oneof_string = 11; + } + + oneof bar { + int64 oneof_int64 = 13; + bytes oneof_bytes = 14; + } + + message Nested { + oneof foo { + int32 oneof_int32 = 10; + string b = 11; + } + } +} diff --git a/third_party/upb/tests/test.proto.pb b/third_party/upb/tests/test.proto.pb new file mode 100644 index 00000000000..a587fb18250 Binary files /dev/null and b/third_party/upb/tests/test.proto.pb differ diff --git a/third_party/upb/tests/test_cpp.cc b/third_party/upb/tests/test_cpp.cc new file mode 100644 index 00000000000..abbafdabc8f --- /dev/null +++ b/third_party/upb/tests/test_cpp.cc @@ -0,0 +1,957 @@ +/* + * + * Tests for C++ wrappers. + */ + +#include +#include + +#include +#include +#include +#include + +#include "tests/test_cpp.upbdefs.h" +#include "tests/upb_test.h" +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/pb/textprinter.h" +#include "upb/port_def.inc" +#include "upb/upb.h" + +template +void AssertInsert(T* const container, const typename T::value_type& val) { + bool inserted = container->insert(val).second; + ASSERT(inserted); +} + +// +// Tests for registering and calling handlers in all their variants. +// This test code is very repetitive because we have to declare each +// handler function variant separately, and they all have different +// signatures so it does not lend itself well to templates. +// +// We test three handler types: +// StartMessage (no data params) +// Int32 (1 data param (int32_t)) +// String Buf (2 data params (const char*, size_t)) +// +// For each handler type we test all 8 handler variants: +// (handler data?) x (function/method) x (returns {void, success}) +// +// The one notable thing we don't test at the moment is +// StartSequence/StartString handlers: these are different from StartMessage() +// in that they return void* for the sub-closure. But this is exercised in +// other tests. +// + +static const int kExpectedHandlerData = 1232323; + +class StringBufTesterBase { + public: + static const int kFieldNumber = 3; + + StringBufTesterBase() : seen_(false), handler_data_val_(0) {} + + void CallAndVerify(upb::Sink sink, upb::FieldDefPtr f) { + upb_selector_t start; + ASSERT(upb_handlers_getselector(f.ptr(), UPB_HANDLER_STARTSTR, &start)); + upb_selector_t str; + ASSERT(upb_handlers_getselector(f.ptr(), UPB_HANDLER_STRING, &str)); + + ASSERT(!seen_); + upb::Sink sub; + sink.StartMessage(); + sink.StartString(start, 0, &sub); + size_t ret = sub.PutStringBuffer(str, &buf_, 5, &handle_); + ASSERT(seen_); + ASSERT(len_ == 5); + ASSERT(ret == 5); + ASSERT(handler_data_val_ == kExpectedHandlerData); + } + + protected: + bool seen_; + int handler_data_val_; + size_t len_; + char buf_; + upb_bufhandle handle_; +}; + +// Test 8 combinations of: +// (handler data?) x (buffer handle?) x (function/method) +// +// Then we add one test each for this variation: to prevent combinatorial +// explosion of these tests we don't test the full 16 combinations, but +// rely on our knowledge that the implementation processes the return wrapping +// in a second separate and independent stage: +// +// (function/method) + +class StringBufTesterVoidMethodNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler(const char *buf, size_t len) { + ASSERT(buf == &buf_); + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidMethodNoHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodNoHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler(const char *buf, size_t len, const upb_bufhandle* handle) { + ASSERT(buf == &buf_); + ASSERT(handle == &handle_); + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidMethodWithHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodWithHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd, const char *buf, size_t len) { + ASSERT(buf == &buf_); + handler_data_val_ = *hd; + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidMethodWithHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodWithHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd, const char* buf, size_t len, + const upb_bufhandle* handle) { + ASSERT(buf == &buf_); + ASSERT(handle == &handle_); + handler_data_val_ = *hd; + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidFunctionNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static void Handler(ME* t, const char *buf, size_t len) { + ASSERT(buf == &t->buf_); + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterVoidFunctionNoHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionNoHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static void Handler(ME* t, const char* buf, size_t len, + const upb_bufhandle* handle) { + ASSERT(buf == &t->buf_); + ASSERT(handle == &t->handle_); + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterVoidFunctionWithHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionWithHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd, const char *buf, size_t len) { + ASSERT(buf == &t->buf_); + t->handler_data_val_ = *hd; + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterVoidFunctionWithHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionWithHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd, const char* buf, size_t len, + const upb_bufhandle* handle) { + ASSERT(buf == &t->buf_); + ASSERT(handle == &t->handle_); + t->handler_data_val_ = *hd; + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterSizeTMethodNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterSizeTMethodNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + size_t Handler(const char *buf, size_t len) { + ASSERT(buf == &buf_); + seen_ = true; + len_ = len; + return len; + } +}; + +class StringBufTesterBoolMethodNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterBoolMethodNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + bool Handler(const char *buf, size_t len) { + ASSERT(buf == &buf_); + seen_ = true; + len_ = len; + return true; + } +}; + +class StartMsgTesterBase { + public: + // We don't need the FieldDef it will create, but the test harness still + // requires that we provide one. + static const int kFieldNumber = 3; + + StartMsgTesterBase() : seen_(false), handler_data_val_(0) {} + + void CallAndVerify(upb::Sink sink, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(!seen_); + sink.StartMessage(); + ASSERT(seen_); + ASSERT(handler_data_val_ == kExpectedHandlerData); + } + + protected: + bool seen_; + int handler_data_val_; +}; + +// Test all 8 combinations of: +// (handler data?) x (function/method) x (returns {void, bool}) + +class StartMsgTesterVoidFunctionNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + //static void Handler(ME* t) { + static void Handler(ME* t) { + t->seen_ = true; + } +}; + +class StartMsgTesterBoolFunctionNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static bool Handler(ME* t) { + t->seen_ = true; + return true; + } +}; + +class StartMsgTesterVoidMethodNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler() { + seen_ = true; + } +}; + +class StartMsgTesterBoolMethodNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + bool Handler() { + seen_ = true; + return true; + } +}; + +class StartMsgTesterVoidFunctionWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd) { + t->handler_data_val_ = *hd; + t->seen_ = true; + } +}; + +class StartMsgTesterBoolFunctionWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static bool Handler(ME* t, const int* hd) { + t->handler_data_val_ = *hd; + t->seen_ = true; + return true; + } +}; + +class StartMsgTesterVoidMethodWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd) { + handler_data_val_ = *hd; + seen_ = true; + } +}; + +class StartMsgTesterBoolMethodWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + bool Handler(const int* hd) { + handler_data_val_ = *hd; + seen_ = true; + return true; + } +}; + +class Int32ValueTesterBase { + public: + static const int kFieldNumber = 1; + + Int32ValueTesterBase() : seen_(false), val_(0), handler_data_val_(0) {} + + void CallAndVerify(upb::Sink sink, upb::FieldDefPtr f) { + upb_selector_t s; + ASSERT(upb_handlers_getselector(f.ptr(), UPB_HANDLER_INT32, &s)); + + ASSERT(!seen_); + sink.PutInt32(s, 5); + ASSERT(seen_); + ASSERT(handler_data_val_ == kExpectedHandlerData); + ASSERT(val_ == 5); + } + + protected: + bool seen_; + int32_t val_; + int handler_data_val_; +}; + +// Test all 8 combinations of: +// (handler data?) x (function/method) x (returns {void, bool}) + +class ValueTesterInt32VoidFunctionNoHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static void Handler(ME* t, int32_t val) { + t->val_ = val; + t->seen_ = true; + } +}; + +class ValueTesterInt32BoolFunctionNoHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static bool Handler(ME* t, int32_t val) { + t->val_ = val; + t->seen_ = true; + return true; + } +}; + +class ValueTesterInt32VoidMethodNoHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler(int32_t val) { + val_ = val; + seen_ = true; + } +}; + +class ValueTesterInt32BoolMethodNoHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + bool Handler(int32_t val) { + val_ = val; + seen_ = true; + return true; + } +}; + +class ValueTesterInt32VoidFunctionWithHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd, int32_t val) { + t->val_ = val; + t->handler_data_val_ = *hd; + t->seen_ = true; + } +}; + +class ValueTesterInt32BoolFunctionWithHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static bool Handler(ME* t, const int* hd, int32_t val) { + t->val_ = val; + t->handler_data_val_ = *hd; + t->seen_ = true; + return true; + } +}; + +class ValueTesterInt32VoidMethodWithHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd, int32_t val) { + val_ = val; + handler_data_val_ = *hd; + seen_ = true; + } +}; + +class ValueTesterInt32BoolMethodWithHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + bool Handler(const int* hd, int32_t val) { + val_ = val; + handler_data_val_ = *hd; + seen_ = true; + return true; + } +}; + +template +void RegisterHandlers(const void* closure, upb::Handlers* h_ptr) { + T* tester = const_cast(static_cast(closure)); + upb::HandlersPtr h(h_ptr); + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(T::kFieldNumber); + ASSERT(f); + tester->Register(h, f); +} + +template +void TestHandler() { + T tester; + upb::SymbolTable symtab; + upb::HandlerCache cache(&RegisterHandlers, &tester); + upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + ASSERT(md); + upb::FieldDefPtr f = md.FindFieldByNumber(T::kFieldNumber); + ASSERT(f); + + const upb::Handlers* h = cache.Get(md); + + upb::Sink sink(h, &tester); + tester.CallAndVerify(sink, f); +} + +class T1 {}; +class T2 {}; + +template +void DoNothingHandler(C* closure) { + UPB_UNUSED(closure); +} + +template +void DoNothingInt32Handler(C* closure, int32_t val) { + UPB_UNUSED(closure); + UPB_UNUSED(val); +} + +template +class DoNothingStartHandler { + public: + // We wrap these functions inside of a class for a somewhat annoying reason. + // UpbMakeHandler() is a macro, so we can't say + // UpbMakeHandler(DoNothingStartHandler) + // + // because otherwise the preprocessor gets confused at the comma and tries to + // make it two macro arguments. The usual solution doesn't work either: + // UpbMakeHandler((DoNothingStartHandler)) + // + // If we do that the macro expands correctly, but then it tries to pass that + // parenthesized expression as a template parameter, ie. Type<(F)>, which + // isn't legal C++ (Clang will compile it but complains with + // warning: address non-type template argument cannot be surrounded by + // parentheses + // + // This two-level thing allows us to effectively pass two template parameters, + // but without any commas: + // UpbMakeHandler(DoNothingStartHandler::Handler) + template + static R* Handler(C* closure) { + UPB_UNUSED(closure); + return NULL; + } + + template + static R* String(C* closure, size_t size_len) { + UPB_UNUSED(closure); + UPB_UNUSED(size_len); + return NULL; + } +}; + +template +void DoNothingStringBufHandler(C* closure, const char *buf, size_t len) { + UPB_UNUSED(closure); + UPB_UNUSED(buf); + UPB_UNUSED(len); +} + +template +void DoNothingEndMessageHandler(C* closure, upb_status *status) { + UPB_UNUSED(closure); + UPB_UNUSED(status); +} + +void RegisterMismatchedTypes(const void* closure, upb::Handlers* h_ptr) { + upb::HandlersPtr h(h_ptr); + + upb::MessageDefPtr md(h.message_def()); + ASSERT(md); + upb::FieldDefPtr i32 = md.FindFieldByName("i32"); + upb::FieldDefPtr r_i32 = md.FindFieldByName("r_i32"); + upb::FieldDefPtr str = md.FindFieldByName("str"); + upb::FieldDefPtr r_str = md.FindFieldByName("r_str"); + upb::FieldDefPtr msg = md.FindFieldByName("msg"); + upb::FieldDefPtr r_msg = md.FindFieldByName("r_msg"); + ASSERT(i32); + ASSERT(r_i32); + ASSERT(str); + ASSERT(r_str); + ASSERT(msg); + ASSERT(r_msg); + + // Establish T1 as the top-level closure type. + ASSERT(h.SetInt32Handler(i32, UpbMakeHandler(DoNothingInt32Handler))); + + // Now any other attempt to set another handler with T2 as the top-level + // closure should fail. But setting these same handlers with T1 as the + // top-level closure will succeed. + ASSERT(!h.SetStartMessageHandler(UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(DoNothingHandler))); + + ASSERT( + !h.SetEndMessageHandler(UpbMakeHandler(DoNothingEndMessageHandler))); + ASSERT( + h.SetEndMessageHandler(UpbMakeHandler(DoNothingEndMessageHandler))); + + ASSERT(!h.SetStartStringHandler( + str, UpbMakeHandler(DoNothingStartHandler::String))); + ASSERT(h.SetStartStringHandler( + str, UpbMakeHandler(DoNothingStartHandler::String))); + + ASSERT(!h.SetEndStringHandler(str, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndStringHandler(str, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSubMessageHandler( + msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSubMessageHandler( + msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT( + !h.SetEndSubMessageHandler(msg, UpbMakeHandler(DoNothingHandler))); + ASSERT( + h.SetEndSubMessageHandler(msg, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSequenceHandler( + r_i32, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSequenceHandler( + r_i32, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSequenceHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSequenceHandler( + r_msg, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSequenceHandler( + r_msg, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSequenceHandler( + r_str, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSequenceHandler( + r_str, UpbMakeHandler(DoNothingHandler))); + + // By setting T1 as the return type for the Start* handlers we have + // established T1 as the type of the sequence and string frames. + // Setting callbacks that use T2 should fail, but T1 should succeed. + ASSERT( + !h.SetStringHandler(str, UpbMakeHandler(DoNothingStringBufHandler))); + ASSERT( + h.SetStringHandler(str, UpbMakeHandler(DoNothingStringBufHandler))); + + ASSERT(!h.SetInt32Handler(r_i32, UpbMakeHandler(DoNothingInt32Handler))); + ASSERT(h.SetInt32Handler(r_i32, UpbMakeHandler(DoNothingInt32Handler))); + + ASSERT(!h.SetStartSubMessageHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSubMessageHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSubMessageHandler(r_msg, + UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSubMessageHandler(r_msg, + UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + ASSERT(h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + + ASSERT( + !h.SetEndStringHandler(r_str, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndStringHandler(r_str, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStringHandler(r_str, + UpbMakeHandler(DoNothingStringBufHandler))); + ASSERT(h.SetStringHandler(r_str, + UpbMakeHandler(DoNothingStringBufHandler))); +} + +void RegisterMismatchedTypes2(const void* closure, upb::Handlers* h_ptr) { + upb::HandlersPtr h(h_ptr); + + upb::MessageDefPtr md(h.message_def()); + ASSERT(md); + upb::FieldDefPtr i32 = md.FindFieldByName("i32"); + upb::FieldDefPtr r_i32 = md.FindFieldByName("r_i32"); + upb::FieldDefPtr str = md.FindFieldByName("str"); + upb::FieldDefPtr r_str = md.FindFieldByName("r_str"); + upb::FieldDefPtr msg = md.FindFieldByName("msg"); + upb::FieldDefPtr r_msg = md.FindFieldByName("r_msg"); + ASSERT(i32); + ASSERT(r_i32); + ASSERT(str); + ASSERT(r_str); + ASSERT(msg); + ASSERT(r_msg); + + // For our second test we do the same in reverse. We directly set the type of + // the frame and then observe failures at registering a Start* handler that + // returns a different type. + + // First establish the type of a sequence frame directly. + ASSERT(h.SetInt32Handler(r_i32, UpbMakeHandler(DoNothingInt32Handler))); + + // Now setting a StartSequence callback that returns a different type should + // fail. + ASSERT(!h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + + // Establish a string frame directly. + ASSERT(h.SetStringHandler(r_str, + UpbMakeHandler(DoNothingStringBufHandler))); + + // Fail setting a StartString callback that returns a different type. + ASSERT(!h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + ASSERT(h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + + // The previous established T1 as the frame for the r_str sequence. + ASSERT(!h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); +} + +void TestMismatchedTypes() { + // First create a schema for our test. + upb::SymbolTable symtab; + upb::HandlerCache handler_cache(&RegisterMismatchedTypes, nullptr); + upb::HandlerCache handler_cache2(&RegisterMismatchedTypes2, nullptr); + const upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + + // Now test the type-checking in handler registration. + handler_cache.Get(md); + handler_cache2.Get(md); +} + +class IntIncrementer { + public: + explicit IntIncrementer(int* x) : x_(x) { (*x_)++; } + ~IntIncrementer() { (*x_)--; } + + static void Handler(void* closure, const IntIncrementer* incrementer, + int32_t x) { + UPB_UNUSED(closure); + UPB_UNUSED(incrementer); + UPB_UNUSED(x); + } + + private: + int* x_; +}; + +void RegisterIncrementor(const void* closure, upb::Handlers* h_ptr) { + const int* x = static_cast(closure); + upb::HandlersPtr h(h_ptr); + upb::FieldDefPtr f = h.message_def().FindFieldByName("i32"); + h.SetInt32Handler(f, UpbBind(&IntIncrementer::Handler, + new IntIncrementer(const_cast(x)))); +} + +void TestHandlerDataDestruction() { + int x = 0; + + { + upb::SymbolTable symtab; + upb::HandlerCache cache(&RegisterIncrementor, &x); + upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + cache.Get(md); + ASSERT(x == 1); + } + + ASSERT(x == 0); +} + +void TestIteration() { + upb::SymbolTable symtab; + upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + + // Test range-based for on both fields and oneofs (with the iterator adaptor). + int field_count = 0; + for (auto field : md.fields()) { + UPB_UNUSED(field); + field_count++; + } + ASSERT(field_count == md.field_count()); + + int oneof_count = 0; + for (auto oneof : md.oneofs()) { + UPB_UNUSED(oneof); + oneof_count++; + } + ASSERT(oneof_count == md.oneof_count()); +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + + TestMismatchedTypes(); + + TestHandlerDataDestruction(); + TestIteration(); + + return 0; +} + +} diff --git a/third_party/upb/tests/test_cpp.proto b/third_party/upb/tests/test_cpp.proto new file mode 100644 index 00000000000..f890350ece1 --- /dev/null +++ b/third_party/upb/tests/test_cpp.proto @@ -0,0 +1,12 @@ +syntax = "proto2"; + +package upb.test; + +message TestMessage { + optional int32 i32 = 1; + repeated int32 r_i32 = 2; + optional string str = 3; + repeated string r_str = 4; + optional TestMessage msg = 5; + repeated TestMessage r_msg = 6; +} diff --git a/third_party/upb/tests/test_table.cc b/third_party/upb/tests/test_table.cc new file mode 100644 index 00000000000..063eb68cee6 --- /dev/null +++ b/third_party/upb/tests/test_table.cc @@ -0,0 +1,679 @@ +/* + * + * Tests for upb_table. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tests/upb_test.h" +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +// Convenience interface for C++. We don't put this in upb itself because +// the table is not exposed to users. + +namespace upb { + +template upb_value MakeUpbValue(T val); +template T GetUpbValue(upb_value val); +template upb_ctype_t GetUpbValueType(); + +#define FUNCS(name, type_t, enumval) \ + template<> upb_value MakeUpbValue(type_t val) { return upb_value_ ## name(val); } \ + template<> type_t GetUpbValue(upb_value val) { return upb_value_get ## name(val); } \ + template<> upb_ctype_t GetUpbValueType() { return enumval; } + +FUNCS(int32, int32_t, UPB_CTYPE_INT32) +FUNCS(int64, int64_t, UPB_CTYPE_INT64) +FUNCS(uint32, uint32_t, UPB_CTYPE_UINT32) +FUNCS(uint64, uint64_t, UPB_CTYPE_UINT64) +FUNCS(bool, bool, UPB_CTYPE_BOOL) +FUNCS(cstr, char*, UPB_CTYPE_CSTR) +FUNCS(ptr, void*, UPB_CTYPE_PTR) +FUNCS(constptr, const void*, UPB_CTYPE_CONSTPTR) +FUNCS(fptr, upb_func*, UPB_CTYPE_FPTR) + +#undef FUNCS + +class IntTable { + public: + IntTable(upb_ctype_t value_type) { upb_inttable_init(&table_, value_type); } + ~IntTable() { upb_inttable_uninit(&table_); } + + size_t count() { return upb_inttable_count(&table_); } + + bool Insert(uintptr_t key, upb_value val) { + return upb_inttable_insert(&table_, key, val); + } + + bool Replace(uintptr_t key, upb_value val) { + return upb_inttable_replace(&table_, key, val); + } + + std::pair Remove(uintptr_t key) { + std::pair ret; + ret.first = upb_inttable_remove(&table_, key, &ret.second); + return ret; + } + + std::pair Lookup(uintptr_t key) const { + std::pair ret; + ret.first = upb_inttable_lookup(&table_, key, &ret.second); + return ret; + } + + std::pair Lookup32(uint32_t key) const { + std::pair ret; + ret.first = upb_inttable_lookup32(&table_, key, &ret.second); + return ret; + } + + void Compact() { upb_inttable_compact(&table_); } + + class iterator : public std::iterator > { + public: + explicit iterator(IntTable* table) { + upb_inttable_begin(&iter_, &table->table_); + } + + static iterator end(IntTable* table) { + iterator iter(table); + upb_inttable_iter_setdone(&iter.iter_); + return iter; + } + + void operator++() { + return upb_inttable_next(&iter_); + } + + std::pair operator*() const { + std::pair ret; + ret.first = upb_inttable_iter_key(&iter_); + ret.second = upb_inttable_iter_value(&iter_); + return ret; + } + + bool operator==(const iterator& other) const { + return upb_inttable_iter_isequal(&iter_, &other.iter_); + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + + private: + upb_inttable_iter iter_; + }; + + upb_inttable table_; +}; + +class StrTable { + public: + StrTable(upb_ctype_t value_type) { upb_strtable_init(&table_, value_type); } + ~StrTable() { upb_strtable_uninit(&table_); } + + size_t count() { return upb_strtable_count(&table_); } + + bool Insert(const std::string& key, upb_value val) { + return upb_strtable_insert2(&table_, key.c_str(), key.size(), val); + } + + std::pair Remove(const std::string& key) { + std::pair ret; + ret.first = + upb_strtable_remove2(&table_, key.c_str(), key.size(), &ret.second); + return ret; + } + + std::pair Lookup(const std::string& key) const { + std::pair ret; + ret.first = + upb_strtable_lookup2(&table_, key.c_str(), key.size(), &ret.second); + return ret; + } + + void Resize(size_t size_lg2) { + upb_strtable_resize(&table_, size_lg2, &upb_alloc_global); + } + + class iterator : public std::iterator > { + public: + explicit iterator(StrTable* table) { + upb_strtable_begin(&iter_, &table->table_); + } + + static iterator end(StrTable* table) { + iterator iter(table); + upb_strtable_iter_setdone(&iter.iter_); + return iter; + } + + void operator++() { + return upb_strtable_next(&iter_); + } + + std::pair operator*() const { + std::pair ret; + ret.first.assign(upb_strtable_iter_key(&iter_)); + ret.second = upb_strtable_iter_value(&iter_); + return ret; + } + + bool operator==(const iterator& other) const { + return upb_strtable_iter_isequal(&iter_, &other.iter_); + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + + private: + upb_strtable_iter iter_; + }; + + upb_strtable table_; +}; + +template class TypedStrTable { + public: + TypedStrTable() : table_(GetUpbValueType()) {} + + size_t count() { return table_.count(); } + + bool Insert(const std::string &key, T val) { + return table_.Insert(key, MakeUpbValue(val)); + } + + std::pair Remove(const std::string& key) { + std::pair found = table_.Remove(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + std::pair Lookup(const std::string& key) const { + std::pair found = table_.Lookup(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + void Resize(size_t size_lg2) { + table_.Resize(size_lg2); + } + + class iterator : public std::iterator > { + public: + explicit iterator(TypedStrTable* table) : iter_(&table->table_) {} + static iterator end(TypedStrTable* table) { + iterator iter(table); + iter.iter_ = StrTable::iterator::end(&table->table_); + return iter; + } + + void operator++() { ++iter_; } + + std::pair operator*() const { + std::pair val = *iter_; + std::pair ret; + ret.first = val.first; + ret.second = GetUpbValue(val.second); + return ret; + } + + bool operator==(const iterator& other) const { + return iter_ == other.iter_; + } + + bool operator!=(const iterator& other) const { + return iter_ != other.iter_; + } + + private: + StrTable::iterator iter_; + }; + + iterator begin() { return iterator(this); } + iterator end() { return iterator::end(this); } + + StrTable table_; +}; + +template class TypedIntTable { + public: + TypedIntTable() : table_(GetUpbValueType()) {} + + size_t count() { return table_.count(); } + + bool Insert(uintptr_t key, T val) { + return table_.Insert(key, MakeUpbValue(val)); + } + + bool Replace(uintptr_t key, T val) { + return table_.Replace(key, MakeUpbValue(val)); + } + + std::pair Remove(uintptr_t key) { + std::pair found = table_.Remove(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + std::pair Lookup(uintptr_t key) const { + std::pair found = table_.Lookup(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + void Compact() { table_.Compact(); } + + class iterator : public std::iterator > { + public: + explicit iterator(TypedIntTable* table) : iter_(&table->table_) {} + static iterator end(TypedIntTable* table) { + return IntTable::iterator::end(&table->table_); + } + + void operator++() { ++iter_; } + + std::pair operator*() const { + std::pair val = *iter_; + std::pair ret; + ret.first = val.first; + ret.second = GetUpbValue(val.second); + return ret; + } + + bool operator==(const iterator& other) const { + return iter_ == other.iter_; + } + + bool operator!=(const iterator& other) const { + return iter_ != other.iter_; + } + + private: + IntTable::iterator iter_; + }; + + iterator begin() { return iterator(this); } + iterator end() { return iterator::end(this); } + + IntTable table_; +}; + +} + +bool benchmark = false; +#define CPU_TIME_PER_TEST 0.5 + +using std::vector; + +double get_usertime() { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec + (usage.ru_utime.tv_usec/1000000.0); +} + +/* num_entries must be a power of 2. */ +void test_strtable(const vector& keys, uint32_t num_to_insert) { + /* Initialize structures. */ + std::map m; + typedef upb::TypedStrTable Table; + Table table; + std::set all; + for(size_t i = 0; i < num_to_insert; i++) { + const std::string& key = keys[i]; + all.insert(key); + table.Insert(key, key[0]); + m[key] = key[0]; + } + + /* Test correctness. */ + for(uint32_t i = 0; i < keys.size(); i++) { + const std::string& key = keys[i]; + std::pair found = table.Lookup(key); + if(m.find(key) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == key[0]); + ASSERT(m[key] == key[0]); + } else { + ASSERT(!found.first); + } + } + + for (Table::iterator it = table.begin(); it != table.end(); ++it) { + std::set::iterator i = all.find((*it).first); + ASSERT(i != all.end()); + all.erase(i); + } + ASSERT(all.empty()); + + // Test iteration with resizes. + + for (int i = 0; i < 10; i++) { + for (Table::iterator it = table.begin(); it != table.end(); ++it) { + // Even if we invalidate the iterator it should only return real elements. + ASSERT((*it).second == m[(*it).first]); + + // Force a resize even though the size isn't changing. + // Also forces the table size to grow so some new buckets end up empty. + int new_lg2 = table.table_.table_.t.size_lg2 + 1; + // Don't use more than 64k tables, to avoid exhausting memory. + new_lg2 = UPB_MIN(new_lg2, 16); + table.Resize(new_lg2); + } + } + +} + +/* num_entries must be a power of 2. */ +void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { + /* Initialize structures. */ + typedef upb::TypedIntTable Table; + Table table; + uint32_t largest_key = 0; + std::map m; + std::unordered_map hm; + for(size_t i = 0; i < num_entries; i++) { + int32_t key = keys[i]; + largest_key = UPB_MAX((int32_t)largest_key, key); + table.Insert(key, key * 2); + m[key] = key*2; + hm[key] = key*2; + } + + /* Test correctness. */ + for(uint32_t i = 0; i <= largest_key; i++) { + std::pair found = table.Lookup(i); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == i*2); + ASSERT(m[i] == i*2); + ASSERT(hm[i] == i*2); + } else { + ASSERT(!found.first); + } + } + + for(uint16_t i = 0; i < num_entries; i += 2) { + std::pair found = table.Remove(keys[i]); + ASSERT(found.first == (m.erase(keys[i]) == 1)); + if (found.first) ASSERT(found.second == (uint32_t)keys[i] * 2); + hm.erase(keys[i]); + m.erase(keys[i]); + } + + ASSERT(table.count() == hm.size()); + + /* Test correctness. */ + for(uint32_t i = 0; i <= largest_key; i++) { + std::pair found = table.Lookup(i); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == i*2); + ASSERT(m[i] == i*2); + ASSERT(hm[i] == i*2); + } else { + ASSERT(!found.first); + } + } + + // Test replace. + for(uint32_t i = 0; i <= largest_key; i++) { + bool replaced = table.Replace(i, i*3); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(replaced); + m[i] = i * 3; + hm[i] = i * 3; + } else { + ASSERT(!replaced); + } + } + + // Compact and test correctness again. + table.Compact(); + for(uint32_t i = 0; i <= largest_key; i++) { + std::pair found = table.Lookup(i); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == i*3); + ASSERT(m[i] == i*3); + ASSERT(hm[i] == i*3); + } else { + ASSERT(!found.first); + } + } + + if(!benchmark) { + return; + } + + printf("%s\n", desc); + + /* Test performance. We only test lookups for keys that are known to exist. */ + uint16_t *rand_order = new uint16_t[num_entries]; + for(uint16_t i = 0; i < num_entries; i++) { + rand_order[i] = i; + } + for(uint16_t i = num_entries - 1; i >= 1; i--) { + uint16_t rand_i = (random() / (double)RAND_MAX) * i; + ASSERT(rand_i <= i); + uint16_t tmp = rand_order[rand_i]; + rand_order[rand_i] = rand_order[i]; + rand_order[i] = tmp; + } + + uintptr_t x = 0; + const int mask = num_entries - 1; + int time_mask = 0xffff; + + printf("upb_inttable(seq): "); + fflush(stdout); + double before = get_usertime(); + unsigned int i; + +#define MAYBE_BREAK \ + if ((i & time_mask) == 0 && (get_usertime() - before) > CPU_TIME_PER_TEST) \ + break; + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[i & mask]; + upb_value v; + bool ok = upb_inttable_lookup32(&table.table_.table_, key, &v); + x += (uintptr_t)ok; + } + double total = get_usertime() - before; + printf("%ld/s\n", (long)(i/total)); + double upb_seq_i = i / 100; // For later percentage calcuation. + + printf("upb_inttable(rand): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + upb_value v; + bool ok = upb_inttable_lookup32(&table.table_.table_, key, &v); + x += (uintptr_t)ok; + } + total = get_usertime() - before; + printf("%ld/s\n", (long)(i/total)); + double upb_rand_i = i / 100; // For later percentage calculation. + + printf("std::map(seq): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[i & mask]; + x += m[key]; + } + total = get_usertime() - before; + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_seq_i); + + printf("std::map(rand): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + x += m[key]; + } + total = get_usertime() - before; + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_rand_i); + + printf("std::unordered_map(seq): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + x += hm[key]; + } + total = get_usertime() - before; + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_seq_i); + + printf("std::unordered_map(rand): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + x += hm[key]; + } + total = get_usertime() - before; + if (x == INT_MAX) abort(); + printf("%ld/s (%0.1f%% of upb)\n\n", (long)(i/total), i / upb_rand_i); + delete[] rand_order; +} + +/* + * This test can't pass right now because the table can't store a value of + * (uint64_t)-1. + */ +void test_int64_max_value() { +/* + typedef upb::TypedIntTable Table; + Table table; + uintptr_t uint64_max = (uint64_t)-1; + table.Insert(1, uint64_max); + std::pair found = table.Lookup(1); + ASSERT(found.first); + ASSERT(found.second == uint64_max); +*/ +} + +int32_t *get_contiguous_keys(int32_t num) { + int32_t *buf = new int32_t[num]; + for(int32_t i = 0; i < num; i++) + buf[i] = i; + return buf; +} + +void test_delete() { + upb_inttable t; + upb_inttable_init(&t, UPB_CTYPE_BOOL); + upb_inttable_insert(&t, 0, upb_value_bool(true)); + upb_inttable_insert(&t, 2, upb_value_bool(true)); + upb_inttable_insert(&t, 4, upb_value_bool(true)); + upb_inttable_compact(&t); + upb_inttable_remove(&t, 0, NULL); + upb_inttable_remove(&t, 2, NULL); + upb_inttable_remove(&t, 4, NULL); + + upb_inttable_iter iter; + for (upb_inttable_begin(&iter, &t); !upb_inttable_done(&iter); + upb_inttable_next(&iter)) { + ASSERT(false); + } + + upb_inttable_uninit(&t); +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "benchmark") == 0) benchmark = true; + } + + vector keys; + keys.push_back("google.protobuf.FileDescriptorSet"); + keys.push_back("google.protobuf.FileDescriptorProto"); + keys.push_back("google.protobuf.DescriptorProto"); + keys.push_back("google.protobuf.DescriptorProto.ExtensionRange"); + keys.push_back("google.protobuf.FieldDescriptorProto"); + keys.push_back("google.protobuf.EnumDescriptorProto"); + keys.push_back("google.protobuf.EnumValueDescriptorProto"); + keys.push_back("google.protobuf.ServiceDescriptorProto"); + keys.push_back("google.protobuf.MethodDescriptorProto"); + keys.push_back("google.protobuf.FileOptions"); + keys.push_back("google.protobuf.MessageOptions"); + keys.push_back("google.protobuf.FieldOptions"); + keys.push_back("google.protobuf.EnumOptions"); + keys.push_back("google.protobuf.EnumValueOptions"); + keys.push_back("google.protobuf.ServiceOptions"); + keys.push_back("google.protobuf.MethodOptions"); + keys.push_back("google.protobuf.UninterpretedOption"); + keys.push_back("google.protobuf.UninterpretedOption.NamePart"); + + for (int i = 0; i < 10; i++) { + test_strtable(keys, 18); + } + + int32_t *keys1 = get_contiguous_keys(8); + test_inttable(keys1, 8, "Table size: 8, keys: 1-8 ===="); + delete[] keys1; + + int32_t *keys2 = get_contiguous_keys(64); + test_inttable(keys2, 64, "Table size: 64, keys: 1-64 ====\n"); + delete[] keys2; + + int32_t *keys3 = get_contiguous_keys(512); + test_inttable(keys3, 512, "Table size: 512, keys: 1-512 ====\n"); + delete[] keys3; + + int32_t *keys4 = new int32_t[64]; + for(int32_t i = 0; i < 64; i++) { + if(i < 32) + keys4[i] = i+1; + else + keys4[i] = 10101+i; + } + test_inttable(keys4, 64, "Table size: 64, keys: 1-32 and 10133-10164 ====\n"); + delete[] keys4; + + test_delete(); + test_int64_max_value(); + + return 0; +} + +} diff --git a/third_party/upb/tests/test_util.h b/third_party/upb/tests/test_util.h new file mode 100644 index 00000000000..485410082df --- /dev/null +++ b/third_party/upb/tests/test_util.h @@ -0,0 +1,230 @@ +/* +** Common functionality for tests. +**/ + +#ifndef UPB_TEST_UTIL_H_ +#define UPB_TEST_UTIL_H_ + +#include +#include +#include "tests/upb_test.h" +#include "upb/sink.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus + +upb_bufhandle global_handle; + +/* A convenience class for parser tests. Provides some useful features: + * + * - can support multiple calls to parse, to test the parser's handling + * of buffer seams. + * + * - can output verbose output about each parse call when requested, for + * ease of debugging. + * + * - can pass NULL for skipped regions of the input if requested. + * + * - allocates and passes a separate buffer for each parsed region, to + * ensure that the parser is not erroneously overreading its buffer. + */ +class VerboseParserEnvironment { + public: + /* Pass verbose=true to print detailed diagnostics to stderr. */ + VerboseParserEnvironment(bool verbose) : verbose_(verbose) {} + + void Reset(const char *buf, size_t len, bool may_skip, bool expect_error) { + buf_ = buf; + len_ = len; + ofs_ = 0; + expect_error_ = expect_error; + end_ok_set_ = false; + skip_until_ = may_skip ? 0 : -1; + skipped_with_null_ = false; + } + + /* The user should call a series of: + * + * Reset(buf, len, may_skip); + * Start() + * ParseBuffer(X); + * ParseBuffer(Y); + * // Repeat ParseBuffer as desired, but last call should pass -1. + * ParseBuffer(-1); + * End(); + */ + + + bool Start() { + if (verbose_) { + fprintf(stderr, "Calling start()\n"); + } + return sink_.Start(len_, &subc_); + } + + bool End() { + if (verbose_) { + fprintf(stderr, "Calling end()\n"); + } + end_ok_ = sink_.End(); + end_ok_set_ = true; + + return end_ok_; + } + + bool CheckConsistency() { + /* If we called end (which we should only do when previous bytes are fully + * accepted), then end() should return true iff there were no errors. */ + if (end_ok_set_ && end_ok_ != status_.ok()) { + fprintf(stderr, "End() status and saw_error didn't match.\n"); + return false; + } + + if (expect_error_ && status_.ok()) { + fprintf(stderr, "Expected error but saw none.\n"); + return false; + } + + if (!status_.ok()) { + if (expect_error_ && verbose_) { + fprintf(stderr, "Encountered error, as expected: %s", + status_.error_message()); + } else if (!expect_error_) { + fprintf(stderr, "Encountered unexpected error: %s", + status_.error_message()); + return false; + } + } + + return true; + } + + bool ParseBuffer(int bytes) { + if (bytes < 0) { + bytes = len_ - ofs_; + } + + ASSERT((size_t)bytes <= (len_ - ofs_)); + + /* Copy buffer into a separate, temporary buffer. + * This is necessary to verify that the parser is not erroneously + * reading outside the specified bounds. */ + char *buf2 = NULL; + + if ((int)(ofs_ + bytes) <= skip_until_) { + skipped_with_null_ = true; + } else { + buf2 = (char*)malloc(bytes); + UPB_ASSERT(buf2); + memcpy(buf2, buf_ + ofs_, bytes); + } + + if (buf2 == NULL && bytes == 0) { + /* Decoders dont' support buf=NULL, bytes=0. */ + return true; + } + + if (verbose_) { + fprintf(stderr, "Calling parse(%u) for bytes %u-%u of the input\n", + (unsigned)bytes, (unsigned)ofs_, (unsigned)(ofs_ + bytes)); + } + + int parsed = sink_.PutBuffer(subc_, buf2, bytes, &global_handle); + free(buf2); + + if (verbose_) { + if (parsed == bytes) { + fprintf(stderr, + "parse(%u) = %u, complete byte count indicates success\n", + (unsigned)bytes, (unsigned)bytes); + } else if (parsed > bytes) { + fprintf(stderr, + "parse(%u) = %u, long byte count indicates success and skip " + "of the next %u bytes\n", + (unsigned)bytes, (unsigned)parsed, (unsigned)(parsed - bytes)); + } else { + fprintf(stderr, + "parse(%u) = %u, short byte count indicates failure; " + "last %u bytes were not consumed\n", + (unsigned)bytes, (unsigned)parsed, (unsigned)(bytes - parsed)); + } + } + + if (!status_.ok()) { + return false; + } + + if (parsed > bytes && skip_until_ >= 0) { + skip_until_ = ofs_ + parsed; + } + + ofs_ += UPB_MIN(parsed, bytes); + + return true; + } + + void ResetBytesSink(upb::BytesSink sink) { + sink_ = sink; + } + + size_t ofs() { return ofs_; } + + bool SkippedWithNull() { return skipped_with_null_; } + + upb::Arena* arena() { return &arena_; } + upb::Status* status() { return &status_; } + + private: + upb::Arena arena_; + upb::Status status_; + upb::BytesSink sink_; + const char* buf_; + size_t len_; + bool verbose_; + size_t ofs_; + void *subc_; + bool expect_error_; + bool end_ok_; + bool end_ok_set_; + + /* When our parse call returns a value greater than the number of bytes + * we passed in, the decoder is indicating to us that the next N bytes + * in the stream are not needed and can be skipped. The user is allowed + * to pass a NULL buffer for those N bytes. + * + * skip_until_ is initially set to 0 if we should do this NULL-buffer + * skipping or -1 if we should not. If we are open to doing NULL-buffer + * skipping and we get an opportunity to do it, we set skip_until to the + * stream offset where we can skip until. The user can then test whether + * this happened by testing SkippedWithNull(). */ + int skip_until_; + bool skipped_with_null_; +}; + +#endif /* __cplusplus */ + +UPB_INLINE char *upb_readfile(const char *filename, size_t *len) { + long size; + char *buf; + FILE *f = fopen(filename, "rb"); + if(!f) return NULL; + if(fseek(f, 0, SEEK_END) != 0) goto error; + size = ftell(f); + if(size < 0) goto error; + if(fseek(f, 0, SEEK_SET) != 0) goto error; + buf = (char*)malloc(size + 1); + if(size && fread(buf, size, 1, f) != 1) goto error; + fclose(f); + if (len) *len = size; + buf[size] = '\0'; + return buf; + +error: + fclose(f); + return NULL; +} + +#include "upb/port_undef.inc" + +#endif /* UPB_TEST_UTIL_H_ */ diff --git a/third_party/upb/tests/testmain.cc b/third_party/upb/tests/testmain.cc new file mode 100644 index 00000000000..97dd7169a46 --- /dev/null +++ b/third_party/upb/tests/testmain.cc @@ -0,0 +1,16 @@ + +#include +#ifdef USE_GOOGLE +#include "base/init_google.h" +#endif + +extern "C" { +int run_tests(int argc, char *argv[]); +} + +int main(int argc, char *argv[]) { +#ifdef USE_GOOGLE + InitGoogle(NULL, &argc, &argv, true); +#endif + run_tests(argc, argv); +} diff --git a/third_party/upb/tests/upb_test.h b/third_party/upb/tests/upb_test.h new file mode 100644 index 00000000000..d4b06883c3e --- /dev/null +++ b/third_party/upb/tests/upb_test.h @@ -0,0 +1,53 @@ + +#ifndef UPB_TEST_H_ +#define UPB_TEST_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int num_assertions = 0; +uint32_t testhash = 0; + +#define PRINT_FAILURE(expr) \ + fprintf(stderr, "Assertion failed: %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, "expr: %s\n", #expr); \ + if (testhash) { \ + fprintf(stderr, "assertion failed running test %x. " \ + "Run with the arg %x to run only this test.\n", \ + testhash, testhash); \ + } + +#define ASSERT(expr) do { \ + ++num_assertions; \ + if (!(expr)) { \ + PRINT_FAILURE(expr) \ + abort(); \ + } \ +} while (0) + +#define ASSERT_NOCOUNT(expr) do { \ + if (!(expr)) { \ + PRINT_FAILURE(expr) \ + abort(); \ + } \ +} while (0) + +#define ASSERT_STATUS(expr, status) do { \ + ++num_assertions; \ + if (!(expr)) { \ + PRINT_FAILURE(expr) \ + fprintf(stderr, "failed status: %s\n", upb_status_errmsg(status)); \ + abort(); \ + } \ +} while (0) + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_DECODER_H_ */ diff --git a/third_party/upb/third_party/lunit/LICENSE b/third_party/upb/third_party/lunit/LICENSE new file mode 100644 index 00000000000..fb720fe4d64 --- /dev/null +++ b/third_party/upb/third_party/lunit/LICENSE @@ -0,0 +1,32 @@ + +Lunit License +------------- + +Lunit is written by Michael Roth and is licensed +under the terms of the MIT license reproduced below. + +======================================================================== + +Copyright (c) 2004-2010 Michael Roth + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +======================================================================== + diff --git a/third_party/upb/third_party/lunit/README.google b/third_party/upb/third_party/lunit/README.google new file mode 100644 index 00000000000..af3e50e7396 --- /dev/null +++ b/third_party/upb/third_party/lunit/README.google @@ -0,0 +1,9 @@ +URL: https://github.com/dcurrie/lunit +Version: 0.5 +License: MIT +License File: LICENSE +Description: +A unit testing library for Lua. + +Local Modifications: +Extracted the two file we actually need from the distribution. diff --git a/third_party/upb/third_party/lunit/console.lua b/third_party/upb/third_party/lunit/console.lua new file mode 100644 index 00000000000..0ff22a42967 --- /dev/null +++ b/third_party/upb/third_party/lunit/console.lua @@ -0,0 +1,156 @@ + +--[[-------------------------------------------------------------------------- + + This file is part of lunit 0.5. + + For Details about lunit look at: http://www.mroth.net/lunit/ + + Author: Michael Roth + + Copyright (c) 2006-2008 Michael Roth + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--]]-------------------------------------------------------------------------- + + + +--[[ + + begin() + run(testcasename, testname) + err(fullname, message, traceback) + fail(fullname, where, message, usermessage) + pass(testcasename, testname) + done() + + Fullname: + testcase.testname + testcase.testname:setupname + testcase.testname:teardownname + +--]] + + +lunit = require "lunit" + +local lunit_console + +if _VERSION >= 'Lua 5.2' then + + lunit_console = setmetatable({},{__index = _ENV}) + _ENV = lunit_console + +else + + module( "lunit-console", package.seeall ) + lunit_console = _M + +end + + + +local function printformat(format, ...) + io.write( string.format(format, ...) ) +end + + +local columns_printed = 0 + +local function writestatus(char) + if columns_printed == 0 then + io.write(" ") + end + if columns_printed == 60 then + io.write("\n ") + columns_printed = 0 + end + io.write(char) + io.flush() + columns_printed = columns_printed + 1 +end + + +local msgs = {} + + +function begin() + local total_tc = 0 + local total_tests = 0 + + msgs = {} -- e + + for tcname in lunit.testcases() do + total_tc = total_tc + 1 + for testname, test in lunit.tests(tcname) do + total_tests = total_tests + 1 + end + end + + printformat("Loaded testsuite with %d tests in %d testcases.\n\n", total_tests, total_tc) +end + + +function run(testcasename, testname) + -- NOP +end + + +function err(fullname, message, traceback) + writestatus("E") + msgs[#msgs+1] = "Error! ("..fullname.."):\n"..message.."\n\t"..table.concat(traceback, "\n\t") .. "\n" +end + + +function fail(fullname, where, message, usermessage) + writestatus("F") + local text = "Failure ("..fullname.."):\n".. + where..": "..message.."\n" + + if usermessage then + text = text .. where..": "..usermessage.."\n" + end + + msgs[#msgs+1] = text +end + + +function pass(testcasename, testname) + writestatus(".") +end + + + +function done() + printformat("\n\n%d Assertions checked.\n", lunit.stats.assertions ) + print() + + for i, msg in ipairs(msgs) do + printformat( "%3d) %s\n", i, msg ) + end + + printformat("Testsuite finished (%d passed, %d failed, %d errors).\n", + lunit.stats.passed, lunit.stats.failed, lunit.stats.errors ) +end + + +return lunit_console + + diff --git a/third_party/upb/third_party/lunit/lunit.lua b/third_party/upb/third_party/lunit/lunit.lua new file mode 100644 index 00000000000..80f43c18e01 --- /dev/null +++ b/third_party/upb/third_party/lunit/lunit.lua @@ -0,0 +1,725 @@ +--[[-------------------------------------------------------------------------- + + This file is part of lunit 0.5. + + For Details about lunit look at: http://www.mroth.net/lunit/ + + Author: Michael Roth + + Copyright (c) 2004, 2006-2010 Michael Roth + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--]]-------------------------------------------------------------------------- + + +local orig_assert = assert + +local pairs = pairs +local ipairs = ipairs +local next = next +local type = type +local error = error +local tostring = tostring +local setmetatable = setmetatable +local pcall = pcall +local xpcall = xpcall +local require = require +local loadfile = loadfile + +local string_sub = string.sub +local string_gsub = string.gsub +local string_format = string.format +local string_lower = string.lower +local string_find = string.find + +local table_concat = table.concat + +local debug_getinfo = debug.getinfo + +local _G = _G + +local lunit + +if _VERSION >= 'Lua 5.2' then + + lunit = {} + _ENV = lunit + +else + + module("lunit") + lunit = _M + +end + + +local __failure__ = {} -- Type tag for failed assertions + +local typenames = { "nil", "boolean", "number", "string", "table", "function", "thread", "userdata" } + + +local traceback_hide -- Traceback function which hides lunit internals +local mypcall -- Protected call to a function with own traceback +do + local _tb_hide = setmetatable( {}, {__mode="k"} ) + + function traceback_hide(func) + _tb_hide[func] = true + end + + local function my_traceback(errobj) + if is_table(errobj) and errobj.type == __failure__ then + local info = debug_getinfo(5, "Sl") -- FIXME: Hardcoded integers are bad... + errobj.where = string_format( "%s:%d", info.short_src, info.currentline) + else + errobj = { msg = tostring(errobj) } + errobj.tb = {} + local i = 2 + while true do + local info = debug_getinfo(i, "Snlf") + if not is_table(info) then + break + end + if not _tb_hide[info.func] then + local line = {} -- Ripped from ldblib.c... + line[#line+1] = string_format("%s:", info.short_src) + if info.currentline > 0 then + line[#line+1] = string_format("%d:", info.currentline) + end + if info.namewhat ~= "" then + line[#line+1] = string_format(" in function '%s'", info.name) + else + if info.what == "main" then + line[#line+1] = " in main chunk" + elseif info.what == "C" or info.what == "tail" then + line[#line+1] = " ?" + else + line[#line+1] = string_format(" in function <%s:%d>", info.short_src, info.linedefined) + end + end + errobj.tb[#errobj.tb+1] = table_concat(line) + end + i = i + 1 + end + end + return errobj + end + + function mypcall(func) + orig_assert( is_function(func) ) + local ok, errobj = xpcall(func, my_traceback) + if not ok then + return errobj + end + end + traceback_hide(mypcall) +end + + +-- Type check functions + +for _, typename in ipairs(typenames) do + lunit["is_"..typename] = function(x) + return type(x) == typename + end +end + +local is_nil = is_nil +local is_boolean = is_boolean +local is_number = is_number +local is_string = is_string +local is_table = is_table +local is_function = is_function +local is_thread = is_thread +local is_userdata = is_userdata + + +local function failure(name, usermsg, defaultmsg, ...) + local errobj = { + type = __failure__, + name = name, + msg = string_format(defaultmsg,...), + usermsg = usermsg + } + error(errobj, 0) +end +traceback_hide( failure ) + + +local function format_arg(arg) + local argtype = type(arg) + if argtype == "string" then + return "'"..arg.."'" + elseif argtype == "number" or argtype == "boolean" or argtype == "nil" then + return tostring(arg) + else + return "["..tostring(arg).."]" + end +end + + +local function selected(map, name) + if not map then + return true + end + + local m = {} + for k,v in pairs(map) do + m[k] = lunitpat2luapat(v) + end + return in_patternmap(m, name) +end + + +function fail(msg) + stats.assertions = stats.assertions + 1 + failure( "fail", msg, "failure" ) +end +traceback_hide( fail ) + + +function assert(assertion, msg) + stats.assertions = stats.assertions + 1 + if not assertion then + failure( "assert", msg, "assertion failed" ) + end + return assertion +end +traceback_hide( assert ) + + +function assert_true(actual, msg) + stats.assertions = stats.assertions + 1 + if actual ~= true then + failure( "assert_true", msg, "true expected but was %s", format_arg(actual) ) + end + return actual +end +traceback_hide( assert_true ) + + +function assert_false(actual, msg) + stats.assertions = stats.assertions + 1 + if actual ~= false then + failure( "assert_false", msg, "false expected but was %s", format_arg(actual) ) + end + return actual +end +traceback_hide( assert_false ) + + +function assert_equal(expected, actual, msg) + stats.assertions = stats.assertions + 1 + if expected ~= actual then + failure( "assert_equal", msg, "expected %s but was %s", format_arg(expected), format_arg(actual) ) + end + return actual +end +traceback_hide( assert_equal ) + + +function assert_not_equal(unexpected, actual, msg) + stats.assertions = stats.assertions + 1 + if unexpected == actual then + failure( "assert_not_equal", msg, "%s not expected but was one", format_arg(unexpected) ) + end + return actual +end +traceback_hide( assert_not_equal ) + + +function assert_match(pattern, actual, msg) + stats.assertions = stats.assertions + 1 + if type(pattern) ~= "string" then + failure( "assert_match", msg, "expected a string as pattern but was %s", format_arg(pattern) ) + end + if type(actual) ~= "string" then + failure( "assert_match", msg, "expected a string to match pattern '%s' but was a %s", pattern, format_arg(actual) ) + end + if not string_find(actual, pattern) then + failure( "assert_match", msg, "expected '%s' to match pattern '%s' but doesn't", actual, pattern ) + end + return actual +end +traceback_hide( assert_match ) + + +function assert_not_match(pattern, actual, msg) + stats.assertions = stats.assertions + 1 + if type(pattern) ~= "string" then + failure( "assert_not_match", msg, "expected a string as pattern but was %s", format_arg(pattern) ) + end + if type(actual) ~= "string" then + failure( "assert_not_match", msg, "expected a string to not match pattern '%s' but was %s", pattern, format_arg(actual) ) + end + if string_find(actual, pattern) then + failure( "assert_not_match", msg, "expected '%s' to not match pattern '%s' but it does", actual, pattern ) + end + return actual +end +traceback_hide( assert_not_match ) + + +function assert_error(msg, func) + stats.assertions = stats.assertions + 1 + if func == nil then + func, msg = msg, nil + end + if type(func) ~= "function" then + failure( "assert_error", msg, "expected a function as last argument but was %s", format_arg(func) ) + end + local ok, errmsg = pcall(func) + if ok then + failure( "assert_error", msg, "error expected but no error occurred" ) + end +end +traceback_hide( assert_error ) + + +function assert_error_match(msg, pattern, func) + stats.assertions = stats.assertions + 1 + if func == nil then + msg, pattern, func = nil, msg, pattern + end + if type(pattern) ~= "string" then + failure( "assert_error_match", msg, "expected the pattern as a string but was %s", format_arg(pattern) ) + end + if type(func) ~= "function" then + failure( "assert_error_match", msg, "expected a function as last argument but was %s", format_arg(func) ) + end + local ok, errmsg = pcall(func) + if ok then + failure( "assert_error_match", msg, "error expected but no error occurred" ) + end + if type(errmsg) ~= "string" then + failure( "assert_error_match", msg, "error as string expected but was %s", format_arg(errmsg) ) + end + if not string_find(errmsg, pattern) then + failure( "assert_error_match", msg, "expected error '%s' to match pattern '%s' but doesn't", errmsg, pattern ) + end +end +traceback_hide( assert_error_match ) + + +function assert_pass(msg, func) + stats.assertions = stats.assertions + 1 + if func == nil then + func, msg = msg, nil + end + if type(func) ~= "function" then + failure( "assert_pass", msg, "expected a function as last argument but was %s", format_arg(func) ) + end + local ok, errmsg = pcall(func) + if not ok then + failure( "assert_pass", msg, "no error expected but error was: '%s'", errmsg ) + end +end +traceback_hide( assert_pass ) + + +-- lunit.assert_typename functions + +for _, typename in ipairs(typenames) do + local assert_typename = "assert_"..typename + lunit[assert_typename] = function(actual, msg) + stats.assertions = stats.assertions + 1 + if type(actual) ~= typename then + failure( assert_typename, msg, "%s expected but was %s", typename, format_arg(actual) ) + end + return actual + end + traceback_hide( lunit[assert_typename] ) +end + + +-- lunit.assert_not_typename functions + +for _, typename in ipairs(typenames) do + local assert_not_typename = "assert_not_"..typename + lunit[assert_not_typename] = function(actual, msg) + stats.assertions = stats.assertions + 1 + if type(actual) == typename then + failure( assert_not_typename, msg, typename.." not expected but was one" ) + end + end + traceback_hide( lunit[assert_not_typename] ) +end + + +function lunit.clearstats() + stats = { + assertions = 0; + passed = 0; + failed = 0; + errors = 0; + } +end + + +local report, reporterrobj +do + local testrunner + + function lunit.setrunner(newrunner) + if not ( is_table(newrunner) or is_nil(newrunner) ) then + return error("lunit.setrunner: Invalid argument", 0) + end + local oldrunner = testrunner + testrunner = newrunner + return oldrunner + end + + function lunit.loadrunner(name) + if not is_string(name) then + return error("lunit.loadrunner: Invalid argument", 0) + end + local ok, runner = pcall( require, name ) + if not ok then + return error("lunit.loadrunner: Can't load test runner: "..runner, 0) + end + return setrunner(runner) + end + + function lunit.getrunner() + return testrunner + end + + function report(event, ...) + local f = testrunner and testrunner[event] + if is_function(f) then + pcall(f, ...) + end + end + + function reporterrobj(context, tcname, testname, errobj) + local fullname = tcname .. "." .. testname + if context == "setup" then + fullname = fullname .. ":" .. setupname(tcname, testname) + elseif context == "teardown" then + fullname = fullname .. ":" .. teardownname(tcname, testname) + end + if errobj.type == __failure__ then + stats.failed = stats.failed + 1 + report("fail", fullname, errobj.where, errobj.msg, errobj.usermsg) + else + stats.errors = stats.errors + 1 + report("err", fullname, errobj.msg, errobj.tb) + end + end +end + + + +local function key_iter(t, k) + return (next(t,k)) +end + + +local testcase +do + -- Array with all registered testcases + local _testcases = {} + + -- Marks a module as a testcase. + -- Applied over a module from module("xyz", lunit.testcase). + function lunit.testcase(m) + orig_assert( is_table(m) ) + --orig_assert( m._M == m ) + orig_assert( is_string(m._NAME) ) + --orig_assert( is_string(m._PACKAGE) ) + + -- Register the module as a testcase + _testcases[m._NAME] = m + + -- Import lunit, fail, assert* and is_* function to the module/testcase + m.lunit = lunit + m.fail = lunit.fail + for funcname, func in pairs(lunit) do + if "assert" == string_sub(funcname, 1, 6) or "is_" == string_sub(funcname, 1, 3) then + m[funcname] = func + end + end + end + + function lunit.module(name,seeall) + local m = {} + if seeall == "seeall" then + setmetatable(m, { __index = _G }) + end + m._NAME = name + lunit.testcase(m) + return m + end + + -- Iterator (testcasename) over all Testcases + function lunit.testcases() + -- Make a copy of testcases to prevent confusing the iterator when + -- new testcase are defined + local _testcases2 = {} + for k,v in pairs(_testcases) do + _testcases2[k] = true + end + return key_iter, _testcases2, nil + end + + function testcase(tcname) + return _testcases[tcname] + end +end + + +do + -- Finds a function in a testcase case insensitive + local function findfuncname(tcname, name) + for key, value in pairs(testcase(tcname)) do + if is_string(key) and is_function(value) and string_lower(key) == name then + return key + end + end + end + + function lunit.setupname(tcname) + return findfuncname(tcname, "setup") + end + + function lunit.teardownname(tcname) + return findfuncname(tcname, "teardown") + end + + -- Iterator over all test names in a testcase. + -- Have to collect the names first in case one of the test + -- functions creates a new global and throws off the iteration. + function lunit.tests(tcname) + local testnames = {} + for key, value in pairs(testcase(tcname)) do + if is_string(key) and is_function(value) then + local lfn = string_lower(key) + if string_sub(lfn, 1, 4) == "test" or string_sub(lfn, -4) == "test" then + testnames[key] = true + end + end + end + return key_iter, testnames, nil + end +end + + + + +function lunit.runtest(tcname, testname) + orig_assert( is_string(tcname) ) + orig_assert( is_string(testname) ) + + if (not getrunner()) then + loadrunner("console") + end + + local function callit(context, func) + if func then + local err = mypcall(func) + if err then + reporterrobj(context, tcname, testname, err) + return false + end + end + return true + end + traceback_hide(callit) + + report("run", tcname, testname) + + local tc = testcase(tcname) + local setup = tc[setupname(tcname)] + local test = tc[testname] + local teardown = tc[teardownname(tcname)] + + local setup_ok = callit( "setup", setup ) + local test_ok = setup_ok and callit( "test", test ) + local teardown_ok = setup_ok and callit( "teardown", teardown ) + + if setup_ok and test_ok and teardown_ok then + stats.passed = stats.passed + 1 + report("pass", tcname, testname) + end +end +traceback_hide(runtest) + + + +function lunit.run(testpatterns) + clearstats() + report("begin") + for testcasename in lunit.testcases() do + -- Run tests in the testcases + for testname in lunit.tests(testcasename) do + if selected(testpatterns, testname) then + runtest(testcasename, testname) + end + end + end + report("done") + return stats +end +traceback_hide(run) + + +function lunit.loadonly() + clearstats() + report("begin") + report("done") + return stats +end + + + + + + + + + +local lunitpat2luapat +do + local conv = { + ["^"] = "%^", + ["$"] = "%$", + ["("] = "%(", + [")"] = "%)", + ["%"] = "%%", + ["."] = "%.", + ["["] = "%[", + ["]"] = "%]", + ["+"] = "%+", + ["-"] = "%-", + ["?"] = ".", + ["*"] = ".*" + } + function lunitpat2luapat(str) + --return "^" .. string.gsub(str, "%W", conv) .. "$" + -- Above was very annoying, if I want to run all the tests having to do with + -- RSS, I want to be able to do "-t rss" not "-t \*rss\*". + return string_gsub(str, "%W", conv) + end +end + + + +local function in_patternmap(map, name) + if map[name] == true then + return true + else + for _, pat in ipairs(map) do + if string_find(name, pat) then + return true + end + end + end + return false +end + + + + + + + + +-- Called from 'lunit' shell script. + +function main(argv) + argv = argv or {} + + -- FIXME: Error handling and error messages aren't nice. + + local function checkarg(optname, arg) + if not is_string(arg) then + return error("lunit.main: option "..optname..": argument missing.", 0) + end + end + + local function loadtestcase(filename) + if not is_string(filename) then + return error("lunit.main: invalid argument") + end + local chunk, err = loadfile(filename) + if err then + return error(err) + else + chunk() + end + end + + local testpatterns = nil + local doloadonly = false + + local i = 0 + while i < #argv do + i = i + 1 + local arg = argv[i] + if arg == "--loadonly" then + doloadonly = true + elseif arg == "--runner" or arg == "-r" then + local optname = arg; i = i + 1; arg = argv[i] + checkarg(optname, arg) + loadrunner(arg) + elseif arg == "--test" or arg == "-t" then + local optname = arg; i = i + 1; arg = argv[i] + checkarg(optname, arg) + testpatterns = testpatterns or {} + testpatterns[#testpatterns+1] = arg + elseif arg == "--help" or arg == "-h" then + print[[ +lunit 0.5 +Copyright (c) 2004-2009 Michael Roth +This program comes WITHOUT WARRANTY OF ANY KIND. + +Usage: lua test [OPTIONS] [--] scripts + +Options: + + -r, --runner RUNNER Testrunner to use, defaults to 'lunit-console'. + -t, --test PATTERN Which tests to run, may contain * or ? wildcards. + --loadonly Only load the tests. + -h, --help Print this help screen. + +Please report bugs to . +]] + return + elseif arg == "--" then + while i < #argv do + i = i + 1; arg = argv[i] + loadtestcase(arg) + end + else + loadtestcase(arg) + end + end + + if doloadonly then + return loadonly() + else + return run(testpatterns) + end +end + +clearstats() + +return lunit diff --git a/third_party/upb/tools/amalgamate.py b/third_party/upb/tools/amalgamate.py new file mode 100755 index 00000000000..bc083b38962 --- /dev/null +++ b/third_party/upb/tools/amalgamate.py @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import sys +import re +import os + +INCLUDE_RE = re.compile('^#include "([^"]*)"$') + +def parse_include(line): + match = INCLUDE_RE.match(line) + return match.groups()[0] if match else None + +class Amalgamator: + def __init__(self, output_path): + self.include_paths = ["."] + self.included = set(["upb/port_def.inc", "upb/port_undef.inc"]) + self.output_h = open(output_path + "upb.h", "w") + self.output_c = open(output_path + "upb.c", "w") + + self.output_c.write("/* Amalgamated source file */\n") + self.output_c.write('#include "upb.h"\n') + self.output_c.write(open("upb/port_def.inc").read()) + + self.output_h.write("/* Amalgamated source file */\n") + self.output_h.write('#include ') + self.output_h.write(open("upb/port_def.inc").read()) + + def add_include_path(self, path): + self.include_paths.append(path) + + def finish(self): + self.output_c.write(open("upb/port_undef.inc").read()) + self.output_h.write(open("upb/port_undef.inc").read()) + + def _process_file(self, infile_name, outfile): + file = None + for path in self.include_paths: + try: + full_path = os.path.join(path, infile_name) + file = open(full_path) + break + except IOError: + pass + if not file: + raise RuntimeError("Couldn't open file " + infile_name) + + for line in file: + include = parse_include(line) + if include is not None and (include.startswith("upb") or + include.startswith("google")): + if include not in self.included: + self.included.add(include) + self._add_header(include) + else: + outfile.write(line) + + def _add_header(self, filename): + self._process_file(filename, self.output_h) + + def add_src(self, filename): + self._process_file(filename, self.output_c) + +# ---- main ---- + +output_path = sys.argv[1] +amalgamator = Amalgamator(output_path) +files = [] + +for arg in sys.argv[2:]: + arg = arg.strip() + if arg.startswith("-I"): + amalgamator.add_include_path(arg[2:]) + elif arg.endswith(".h") or arg.endswith(".inc"): + pass + else: + files.append(arg) + +for filename in files: + amalgamator.add_src(filename) + +amalgamator.finish() diff --git a/third_party/upb/tools/make_cmakelists.py b/third_party/upb/tools/make_cmakelists.py new file mode 100755 index 00000000000..a4923c8da07 --- /dev/null +++ b/third_party/upb/tools/make_cmakelists.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python + +"""TODO(haberman): DO NOT SUBMIT without one-line documentation for make_cmakelists. + +TODO(haberman): DO NOT SUBMIT without a detailed description of make_cmakelists. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import textwrap +import os + +def StripColons(deps): + return map(lambda x: x[1:], deps) + +def IsSourceFile(name): + return name.endswith(".c") or name.endswith(".cc") + +class BuildFileFunctions(object): + def __init__(self, converter): + self.converter = converter + + def _add_deps(self, kwargs, keyword=""): + if "deps" not in kwargs: + return + self.converter.toplevel += "target_link_libraries(%s%s\n %s)\n" % ( + kwargs["name"], + keyword, + "\n ".join(StripColons(kwargs["deps"])) + ) + + def load(self, *args): + pass + + def cc_library(self, **kwargs): + if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator": + return + files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) + found_files = [] + for file in files: + if os.path.isfile(file): + found_files.append(file) + elif os.path.isfile("generated_for_cmake/" + file): + found_files.append("generated_for_cmake/" + file) + else: + print("Warning: no such file: " + file) + + if list(filter(IsSourceFile, files)): + # Has sources, make this a normal library. + self.converter.toplevel += "add_library(%s\n %s)\n" % ( + kwargs["name"], + "\n ".join(found_files) + ) + self._add_deps(kwargs) + else: + # Header-only library, have to do a couple things differently. + # For some info, see: + # http://mariobadr.com/creating-a-header-only-library-with-cmake.html + self.converter.toplevel += "add_library(%s INTERFACE)\n" % ( + kwargs["name"] + ) + self._add_deps(kwargs, " INTERFACE") + + def cc_binary(self, **kwargs): + pass + + def cc_test(self, **kwargs): + # Disable this until we properly support upb_proto_library(). + # self.converter.toplevel += "add_executable(%s\n %s)\n" % ( + # kwargs["name"], + # "\n ".join(kwargs["srcs"]) + # ) + # self.converter.toplevel += "add_test(NAME %s COMMAND %s)\n" % ( + # kwargs["name"], + # kwargs["name"], + # ) + + # if "data" in kwargs: + # for data_dep in kwargs["data"]: + # self.converter.toplevel += textwrap.dedent("""\ + # add_custom_command( + # TARGET %s POST_BUILD + # COMMAND ${CMAKE_COMMAND} -E copy + # ${CMAKE_SOURCE_DIR}/%s + # ${CMAKE_CURRENT_BINARY_DIR}/%s)\n""" % ( + # kwargs["name"], data_dep, data_dep + # )) + + # self._add_deps(kwargs) + pass + + def py_library(self, **kwargs): + pass + + def py_binary(self, **kwargs): + pass + + def lua_cclibrary(self, **kwargs): + pass + + def lua_library(self, **kwargs): + pass + + def lua_binary(self, **kwargs): + pass + + def lua_test(self, **kwargs): + pass + + def sh_test(self, **kwargs): + pass + + def make_shell_script(self, **kwargs): + pass + + def exports_files(self, files, **kwargs): + pass + + def proto_library(self, **kwargs): + pass + + def generated_file_staleness_test(self, **kwargs): + pass + + def upb_amalgamation(self, **kwargs): + pass + + def upb_proto_library(self, **kwargs): + pass + + def upb_proto_reflection_library(self, **kwargs): + pass + + def upb_proto_srcs(self, **kwargs): + pass + + def genrule(self, **kwargs): + pass + + def config_setting(self, **kwargs): + pass + + def select(self, arg_dict): + return [] + + def glob(self, *args): + return [] + + def licenses(self, *args): + pass + + def filegroup(self, **kwargs): + pass + + def map_dep(self, arg): + return arg + + +class WorkspaceFileFunctions(object): + def __init__(self, converter): + self.converter = converter + + def load(self, *args): + pass + + def workspace(self, **kwargs): + self.converter.prelude += "project(%s)\n" % (kwargs["name"]) + + def http_archive(self, **kwargs): + pass + + def git_repository(self, **kwargs): + pass + + def bazel_version_repository(self, **kwargs): + pass + + def upb_deps(self): + pass + + +class Converter(object): + def __init__(self): + self.prelude = "" + self.toplevel = "" + self.if_lua = "" + + def convert(self): + return self.template % { + "prelude": converter.prelude, + "toplevel": converter.toplevel, + } + + template = textwrap.dedent("""\ + # This file was generated from BUILD using tools/make_cmakelists.py. + + cmake_minimum_required(VERSION 3.1) + + if(${CMAKE_VERSION} VERSION_LESS 3.12) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) + else() + cmake_policy(VERSION 3.12) + endif() + + cmake_minimum_required (VERSION 3.0) + cmake_policy(SET CMP0048 NEW) + + %(prelude)s + + # Prevent CMake from setting -rdynamic on Linux (!!). + SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") + SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + + # Set default build type. + if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) + endif() + + # When using Ninja, compiler output won't be colorized without this. + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS) + if(SUPPORTS_COLOR_ALWAYS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + endif() + + # Implement ASAN/UBSAN options + if(UPB_ENABLE_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + endif() + + if(UPB_ENABLE_UBSAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + endif() + + include_directories(.) + include_directories(generated_for_cmake) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) + + if(APPLE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") + elseif(UNIX) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") + endif() + + enable_testing() + + %(toplevel)s + + """) + +data = {} +converter = Converter() + +def GetDict(obj): + ret = {} + for k in dir(obj): + if not k.startswith("_"): + ret[k] = getattr(obj, k); + return ret + +globs = GetDict(converter) + +exec(open("WORKSPACE").read(), GetDict(WorkspaceFileFunctions(converter))) +exec(open("BUILD").read(), GetDict(BuildFileFunctions(converter))) + +with open(sys.argv[1], "w") as f: + f.write(converter.convert()) diff --git a/third_party/upb/tools/staleness_test.py b/third_party/upb/tools/staleness_test.py new file mode 100644 index 00000000000..045cd1a319d --- /dev/null +++ b/third_party/upb/tools/staleness_test.py @@ -0,0 +1,30 @@ +"""The py_test() script for generated_file_staleness_test() rules. + +Note that this file is preprocessed! The INSERT_<...> text below is replaced +with the actual list of files before we actually run the script. +""" + +from __future__ import absolute_import + +from tools import staleness_test_lib +import unittest +import sys + +file_list = """ + INSERT_FILE_LIST_HERE +""".split() + +config = staleness_test_lib.Config(file_list) + + +class TestFilesMatch(unittest.TestCase): + + def testFilesMatch(self): + errors = staleness_test_lib.CheckFilesMatch(config) + self.assertFalse(errors, errors) + + +if len(sys.argv) > 1 and sys.argv[1] == "--fix": + staleness_test_lib.FixFiles(config) +else: + unittest.main() diff --git a/third_party/upb/tools/staleness_test_lib.py b/third_party/upb/tools/staleness_test_lib.py new file mode 100644 index 00000000000..6d5c3d30683 --- /dev/null +++ b/third_party/upb/tools/staleness_test_lib.py @@ -0,0 +1,158 @@ +"""Shared code for validating generated_file_staleness_test() rules. + +This code is used by test scripts generated from +generated_file_staleness_test() rules. +""" + +from __future__ import absolute_import +from __future__ import print_function + +import os +from shutil import copyfile + + +class _FilePair(object): + """Represents a single (target, generated) file pair.""" + + def __init__(self, target, generated): + self.target = target + self.generated = generated + + +class Config(object): + """Represents the configuration for a single staleness test target.""" + + def __init__(self, file_list): + # Duplicate to avoid modifying our arguments. + file_list = list(file_list) + + # The file list contains a few other bits of information at the end. + # This is packed by the code in build_defs.bzl. + self.target_name = file_list.pop() + self.package_name = file_list.pop() + self.pattern = file_list.pop() + + self.file_list = file_list + + +def _GetFilePairs(config): + """Generates the list of file pairs. + + Args: + config: a Config object representing this target's config. + + Returns: + A list of _FilePair objects. + """ + + ret = [] + + has_bazel_genfiles = os.path.exists("bazel-genfiles") + + for filename in config.file_list: + target = os.path.join(config.package_name, filename) + generated = os.path.join(config.package_name, config.pattern % filename) + if has_bazel_genfiles: + generated = os.path.join("bazel-genfiles", generated) + + # Generated files should always exist. Blaze should guarantee this before + # we are run. + if not os.path.isfile(generated): + print("Generated file '%s' does not exist." % generated) + print("Please run this command to generate it:") + print(" bazel build %s:%s" % (config.package_name, config.target_name)) + ret.append(_FilePair(target, generated)) + + return ret + + +def _GetMissingAndStaleFiles(file_pairs): + """Generates lists of missing and stale files. + + Args: + file_pairs: a list of _FilePair objects. + + Returns: + missing_files: a list of _FilePair objects representing missing files. + These target files do not exist at all. + stale_files: a list of _FilePair objects representing stale files. + These target files exist but have stale contents. + """ + + missing_files = [] + stale_files = [] + + for pair in file_pairs: + if not os.path.isfile(pair.target): + missing_files.append(pair) + continue + + generated = open(pair.generated).read() + target = open(pair.target).read() + if generated != target: + stale_files.append(pair) + + return missing_files, stale_files + + +def _CopyFiles(file_pairs): + """Copies all generated files to the corresponding target file. + + The target files must be writable already. + + Args: + file_pairs: a list of _FilePair objects that we want to copy. + """ + + for pair in file_pairs: + target_dir = os.path.dirname(pair.target) + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + copyfile(pair.generated, pair.target) + + +def FixFiles(config): + """Implements the --fix option: overwrites missing or out-of-date files. + + Args: + config: the Config object for this test. + """ + + file_pairs = _GetFilePairs(config) + missing_files, stale_files = _GetMissingAndStaleFiles(file_pairs) + + _CopyFiles(stale_files + missing_files) + + +def CheckFilesMatch(config): + """Checks whether each target file matches the corresponding generated file. + + Args: + config: the Config object for this test. + + Returns: + None if everything matches, otherwise a string error message. + """ + + diff_errors = [] + + file_pairs = _GetFilePairs(config) + missing_files, stale_files = _GetMissingAndStaleFiles(file_pairs) + + for pair in missing_files: + diff_errors.append("File %s does not exist" % pair.target) + continue + + for pair in stale_files: + diff_errors.append("File %s is out of date" % pair.target) + + if diff_errors: + error_msg = "Files out of date!\n\n" + error_msg += "To fix run THIS command:\n" + error_msg += " bazel-bin/%s/%s --fix\n\n" % (config.package_name, + config.target_name) + error_msg += "Errors:\n" + error_msg += " " + "\n ".join(diff_errors) + return error_msg + else: + return None diff --git a/third_party/upb/upb/bindings/README b/third_party/upb/upb/bindings/README new file mode 100644 index 00000000000..3e176c943bd --- /dev/null +++ b/third_party/upb/upb/bindings/README @@ -0,0 +1,5 @@ +This directory contains code that interfaces upb with external C/C++ +libraries. Right now this is: + + * upb/bindings/lua: + a Lua extension that exposes upb to Lua programs via the Lua C API. diff --git a/third_party/upb/upb/bindings/lua/def.c b/third_party/upb/upb/bindings/lua/def.c new file mode 100644 index 00000000000..a9ab9a89db0 --- /dev/null +++ b/third_party/upb/upb/bindings/lua/def.c @@ -0,0 +1,766 @@ + +#include +#include +#include +#include +#include "lauxlib.h" +#include "upb/bindings/lua/upb.h" +#include "upb/def.h" + +#define LUPB_ENUMDEF "lupb.enumdef" +#define LUPB_FIELDDEF "lupb.fielddef" +#define LUPB_FILEDEF "lupb.filedef" +#define LUPB_MSGDEF "lupb.msgdef" +#define LUPB_ONEOFDEF "lupb.oneof" +#define LUPB_SYMTAB "lupb.symtab" +#define LUPB_OBJCACHE "lupb.objcache" + +#define CHK(pred) \ + do { \ + upb_status status; \ + upb_status_clear(&status); \ + pred; \ + lupb_checkstatus(L, &status); \ + } while (0) + +/* lupb_wrapper ***************************************************************/ + +/* Wrappers around upb objects. */ + +/* Checks type; if it matches, pulls the pointer out of the wrapper. */ +void *lupb_checkwrapper(lua_State *L, int narg, const char *type) { + void *ud = lua_touserdata(L, narg); + void *ret; + + if (!ud) { + luaL_typerror(L, narg, "upb wrapper"); + } + + memcpy(&ret, ud, sizeof(ret)); + if (!ret) { + luaL_error(L, "called into dead object"); + } + + luaL_checkudata(L, narg, type); + return ret; +} + +void lupb_pushwrapper(lua_State *L, const void *obj, const char *type) { + void *ud; + + if (obj == NULL) { + lua_pushnil(L); + return; + } + + /* Lookup our cache in the registry (we don't put our objects in the registry + * directly because we need our cache to be a weak table). */ + lua_getfield(L, LUA_REGISTRYINDEX, LUPB_OBJCACHE); + UPB_ASSERT(!lua_isnil(L, -1)); /* Should have been created by luaopen_upb. */ + lua_pushlightuserdata(L, (void*)obj); + lua_rawget(L, -2); + /* Stack is now: objcache, cached value. */ + + if (lua_isnil(L, -1)) { + /* Remove bad cached value and push new value. */ + lua_pop(L, 1); + ud = lua_newuserdata(L, sizeof(*ud)); + memcpy(ud, &obj, sizeof(*ud)); + + luaL_getmetatable(L, type); + /* Should have been created by luaopen_upb. */ + lupb_assert(L, !lua_isnil(L, -1)); + lua_setmetatable(L, -2); + + /* Set it in the cache. */ + lua_pushlightuserdata(L, (void*)obj); + lua_pushvalue(L, -2); + lua_rawset(L, -4); + } + + lua_insert(L, -2); + lua_pop(L, 1); +} + +void lupb_msgdef_pushwrapper(lua_State *L, const upb_msgdef *m); +void lupb_oneofdef_pushwrapper(lua_State *L, const upb_oneofdef *o); +static void lupb_enumdef_pushwrapper(lua_State *L, const upb_enumdef *e); + + +/* lupb_fielddef **************************************************************/ + +void lupb_fielddef_pushwrapper(lua_State *L, const upb_fielddef *f) { + lupb_pushwrapper(L, f, LUPB_FIELDDEF); +} + +const upb_fielddef *lupb_fielddef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_FIELDDEF); +} + +static int lupb_fielddef_containingoneof(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_oneofdef_pushwrapper(L, upb_fielddef_containingoneof(f)); + return 1; +} + +static int lupb_fielddef_containingtype(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_msgdef_pushwrapper(L, upb_fielddef_containingtype(f)); + return 1; +} + +static int lupb_fielddef_default(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: + lupb_pushint32(L, upb_fielddef_defaultint32(f)); break; + case UPB_TYPE_INT64: + lupb_pushint64(L, upb_fielddef_defaultint64(f)); break; + case UPB_TYPE_UINT32: + lupb_pushuint32(L, upb_fielddef_defaultuint32(f)); break; + case UPB_TYPE_UINT64: + lupb_pushuint64(L, upb_fielddef_defaultuint64(f)); break; + case UPB_TYPE_DOUBLE: + lua_pushnumber(L, upb_fielddef_defaultdouble(f)); break; + case UPB_TYPE_FLOAT: + lua_pushnumber(L, upb_fielddef_defaultfloat(f)); break; + case UPB_TYPE_BOOL: + lua_pushboolean(L, upb_fielddef_defaultbool(f)); break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + size_t len; + const char *data = upb_fielddef_defaultstr(f, &len); + lua_pushlstring(L, data, len); + break; + } + case UPB_TYPE_MESSAGE: + return luaL_error(L, "Message fields do not have explicit defaults."); + } + return 1; +} + +static int lupb_fielddef_descriptortype(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushnumber(L, upb_fielddef_descriptortype(f)); + return 1; +} + +static int lupb_fielddef_getsel(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + upb_selector_t sel; + if (upb_handlers_getselector(f, luaL_checknumber(L, 2), &sel)) { + lua_pushinteger(L, sel); + return 1; + } else { + return 0; + } +} + +static int lupb_fielddef_hassubdef(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_hassubdef(f)); + return 1; +} + +static int lupb_fielddef_index(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushinteger(L, upb_fielddef_index(f)); + return 1; +} + +static int lupb_fielddef_isextension(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_isextension(f)); + return 1; +} + +static int lupb_fielddef_label(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushinteger(L, upb_fielddef_label(f)); + return 1; +} + +static int lupb_fielddef_lazy(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_lazy(f)); + return 1; +} + +static int lupb_fielddef_name(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushstring(L, upb_fielddef_name(f)); + return 1; +} + +static int lupb_fielddef_number(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + int32_t num = upb_fielddef_number(f); + if (num) + lua_pushinteger(L, num); + else + lua_pushnil(L); + return 1; +} + +static int lupb_fielddef_packed(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_packed(f)); + return 1; +} + +static int lupb_fielddef_msgsubdef(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_msgdef_pushwrapper(L, upb_fielddef_msgsubdef(f)); + return 1; +} + +static int lupb_fielddef_enumsubdef(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_enumdef_pushwrapper(L, upb_fielddef_enumsubdef(f)); + return 1; +} + +static int lupb_fielddef_type(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + if (upb_fielddef_typeisset(f)) + lua_pushinteger(L, upb_fielddef_type(f)); + else + lua_pushnil(L); + return 1; +} + +static const struct luaL_Reg lupb_fielddef_m[] = { + {"containing_oneof", lupb_fielddef_containingoneof}, + {"containing_type", lupb_fielddef_containingtype}, + {"default", lupb_fielddef_default}, + {"descriptor_type", lupb_fielddef_descriptortype}, + {"getsel", lupb_fielddef_getsel}, + {"has_subdef", lupb_fielddef_hassubdef}, + {"index", lupb_fielddef_index}, + {"is_extension", lupb_fielddef_isextension}, + {"label", lupb_fielddef_label}, + {"lazy", lupb_fielddef_lazy}, + {"name", lupb_fielddef_name}, + {"number", lupb_fielddef_number}, + {"packed", lupb_fielddef_packed}, + {"msgsubdef", lupb_fielddef_msgsubdef}, + {"enumsubdef", lupb_fielddef_enumsubdef}, + {"type", lupb_fielddef_type}, + {NULL, NULL} +}; + + +/* lupb_oneofdef **************************************************************/ + +void lupb_oneofdef_pushwrapper(lua_State *L, const upb_oneofdef *o) { + lupb_pushwrapper(L, o, LUPB_ONEOFDEF); +} + +const upb_oneofdef *lupb_oneofdef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_ONEOFDEF); +} + +static int lupb_oneofdef_containingtype(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + lupb_msgdef_pushwrapper(L, upb_oneofdef_containingtype(o)); + return 1; +} + +static int lupb_oneofdef_field(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + int type = lua_type(L, 2); + const upb_fielddef *f; + if (type == LUA_TNUMBER) { + f = upb_oneofdef_itof(o, lua_tointeger(L, 2)); + } else if (type == LUA_TSTRING) { + f = upb_oneofdef_ntofz(o, lua_tostring(L, 2)); + } else { + const char *msg = lua_pushfstring(L, "number or string expected, got %s", + luaL_typename(L, 2)); + return luaL_argerror(L, 2, msg); + } + + lupb_fielddef_pushwrapper(L, f); + return 1; +} + +static int lupb_oneofiter_next(lua_State *L) { + upb_oneof_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_oneof_done(i)) return 0; + lupb_fielddef_pushwrapper(L, upb_oneof_iter_field(i)); + upb_oneof_next(i); + return 1; +} + +static int lupb_oneofdef_fields(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + upb_oneof_iter *i = lua_newuserdata(L, sizeof(upb_oneof_iter)); + upb_oneof_begin(i, o); + /* Need to guarantee that the msgdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_oneofiter_next, 2); + return 1; +} + +static int lupb_oneofdef_len(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + lua_pushinteger(L, upb_oneofdef_numfields(o)); + return 1; +} + +static int lupb_oneofdef_name(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + lua_pushstring(L, upb_oneofdef_name(o)); + return 1; +} + +static const struct luaL_Reg lupb_oneofdef_m[] = { + {"containing_type", lupb_oneofdef_containingtype}, + {"field", lupb_oneofdef_field}, + {"fields", lupb_oneofdef_fields}, + {"name", lupb_oneofdef_name}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_oneofdef_mm[] = { + {"__len", lupb_oneofdef_len}, + {NULL, NULL} +}; + + +/* lupb_msgdef ****************************************************************/ + +typedef struct { + const upb_msgdef *md; +} lupb_msgdef; + +void lupb_msgdef_pushwrapper(lua_State *L, const upb_msgdef *m) { + lupb_pushwrapper(L, m, LUPB_MSGDEF); +} + +const upb_msgdef *lupb_msgdef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_MSGDEF); +} + +static int lupb_msgdef_len(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + lua_pushinteger(L, upb_msgdef_numfields(m)); + return 1; +} + +static int lupb_msgdef_field(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + int type = lua_type(L, 2); + const upb_fielddef *f; + if (type == LUA_TNUMBER) { + f = upb_msgdef_itof(m, lua_tointeger(L, 2)); + } else if (type == LUA_TSTRING) { + f = upb_msgdef_ntofz(m, lua_tostring(L, 2)); + } else { + const char *msg = lua_pushfstring(L, "number or string expected, got %s", + luaL_typename(L, 2)); + return luaL_argerror(L, 2, msg); + } + + lupb_fielddef_pushwrapper(L, f); + return 1; +} + +static int lupb_msgdef_lookupname(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + const upb_fielddef *f; + const upb_oneofdef *o; + if (!upb_msgdef_lookupnamez(m, lua_tostring(L, 2), &f, &o)) { + lua_pushnil(L); + } else if (o) { + lupb_oneofdef_pushwrapper(L, o); + } else { + lupb_fielddef_pushwrapper(L, f); + } + return 1; +} + +static int lupb_msgfielditer_next(lua_State *L) { + upb_msg_field_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_msg_field_done(i)) return 0; + lupb_fielddef_pushwrapper(L, upb_msg_iter_field(i)); + upb_msg_field_next(i); + return 1; +} + +static int lupb_msgdef_fields(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + upb_msg_field_iter *i = lua_newuserdata(L, sizeof(upb_msg_field_iter)); + upb_msg_field_begin(i, m); + /* Need to guarantee that the msgdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_msgfielditer_next, 2); + return 1; +} + +static int lupb_msgoneofiter_next(lua_State *L) { + upb_msg_oneof_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_msg_oneof_done(i)) return 0; + lupb_oneofdef_pushwrapper(L, upb_msg_iter_oneof(i)); + upb_msg_oneof_next(i); + return 1; +} + +static int lupb_msgdef_oneofs(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + upb_msg_oneof_iter *i = lua_newuserdata(L, sizeof(upb_msg_oneof_iter)); + upb_msg_oneof_begin(i, m); + /* Need to guarantee that the msgdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_msgoneofiter_next, 2); + return 1; +} + +static int lupb_msgdef_mapentry(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + lua_pushboolean(L, upb_msgdef_mapentry(m)); + return 1; +} + +static int lupb_msgdef_syntax(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + lua_pushinteger(L, upb_msgdef_syntax(m)); + return 1; +} + +static const struct luaL_Reg lupb_msgdef_mm[] = { + {"__len", lupb_msgdef_len}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_msgdef_m[] = { + {"field", lupb_msgdef_field}, + {"fields", lupb_msgdef_fields}, + {"lookup_name", lupb_msgdef_lookupname}, + {"oneofs", lupb_msgdef_oneofs}, + {"syntax", lupb_msgdef_syntax}, + {"_map_entry", lupb_msgdef_mapentry}, + {NULL, NULL} +}; + + +/* lupb_enumdef ***************************************************************/ + +const upb_enumdef *lupb_enumdef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_ENUMDEF); +} + +static void lupb_enumdef_pushwrapper(lua_State *L, const upb_enumdef *e) { + lupb_pushwrapper(L, e, LUPB_ENUMDEF); +} + +static int lupb_enumdef_len(lua_State *L) { + const upb_enumdef *e = lupb_enumdef_check(L, 1); + lua_pushinteger(L, upb_enumdef_numvals(e)); + return 1; +} + +static int lupb_enumdef_value(lua_State *L) { + const upb_enumdef *e = lupb_enumdef_check(L, 1); + int type = lua_type(L, 2); + if (type == LUA_TNUMBER) { + /* Pushes "nil" for a NULL pointer. */ + int32_t key = lupb_checkint32(L, 2); + lua_pushstring(L, upb_enumdef_iton(e, key)); + } else if (type == LUA_TSTRING) { + const char *key = lua_tostring(L, 2); + int32_t num; + if (upb_enumdef_ntoiz(e, key, &num)) { + lua_pushinteger(L, num); + } else { + lua_pushnil(L); + } + } else { + const char *msg = lua_pushfstring(L, "number or string expected, got %s", + luaL_typename(L, 2)); + return luaL_argerror(L, 2, msg); + } + return 1; +} + +static int lupb_enumiter_next(lua_State *L) { + upb_enum_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_enum_done(i)) return 0; + lua_pushstring(L, upb_enum_iter_name(i)); + lua_pushinteger(L, upb_enum_iter_number(i)); + upb_enum_next(i); + return 2; +} + +static int lupb_enumdef_values(lua_State *L) { + const upb_enumdef *e = lupb_enumdef_check(L, 1); + upb_enum_iter *i = lua_newuserdata(L, sizeof(upb_enum_iter)); + upb_enum_begin(i, e); + /* Need to guarantee that the enumdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_enumiter_next, 2); + return 1; +} + +static const struct luaL_Reg lupb_enumdef_mm[] = { + {"__len", lupb_enumdef_len}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_enumdef_m[] = { + {"value", lupb_enumdef_value}, + {"values", lupb_enumdef_values}, + {NULL, NULL} +}; + + +/* lupb_filedef ***************************************************************/ + +void lupb_filedef_pushwrapper(lua_State *L, const upb_filedef *f) { + lupb_pushwrapper(L, f, LUPB_FILEDEF); +} + +const upb_filedef *lupb_filedef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_FILEDEF); +} + +static int lupb_filedef_dep(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + int index = luaL_checkint(L, 2); + lupb_filedef_pushwrapper(L, upb_filedef_dep(f, index)); + return 1; +} + +static int lupb_filedef_depcount(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_depcount(f)); + return 1; +} + +static int lupb_filedef_enum(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + int index = luaL_checkint(L, 2); + lupb_enumdef_pushwrapper(L, upb_filedef_enum(f, index)); + return 1; +} + +static int lupb_filedef_enumcount(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_enumcount(f)); + return 1; +} + +static int lupb_filedef_msg(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + int index = luaL_checkint(L, 2); + lupb_msgdef_pushwrapper(L, upb_filedef_msg(f, index)); + return 1; +} + +static int lupb_filedef_msgcount(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_msgcount(f)); + return 1; +} + +static int lupb_filedef_name(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushstring(L, upb_filedef_name(f)); + return 1; +} + +static int lupb_filedef_package(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushstring(L, upb_filedef_package(f)); + return 1; +} + +static int lupb_filedef_syntax(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_syntax(f)); + return 1; +} + +static const struct luaL_Reg lupb_filedef_m[] = { + {"dep", lupb_filedef_dep}, + {"depcount", lupb_filedef_depcount}, + {"enum", lupb_filedef_enum}, + {"enumcount", lupb_filedef_enumcount}, + {"msg", lupb_filedef_msg}, + {"msgcount", lupb_filedef_msgcount}, + {"name", lupb_filedef_name}, + {"package", lupb_filedef_package}, + {"syntax", lupb_filedef_syntax}, + {NULL, NULL} +}; + + +/* lupb_symtab ****************************************************************/ + +typedef struct { + upb_symtab *symtab; +} lupb_symtab; + +upb_symtab *lupb_symtab_check(lua_State *L, int narg) { + lupb_symtab *lsymtab = luaL_checkudata(L, narg, LUPB_SYMTAB); + if (!lsymtab->symtab) { + luaL_error(L, "called into dead object"); + } + return lsymtab->symtab; +} + +static int lupb_symtab_new(lua_State *L) { + lupb_symtab *lsymtab = lua_newuserdata(L, sizeof(*lsymtab)); + lsymtab->symtab = upb_symtab_new(); + luaL_getmetatable(L, LUPB_SYMTAB); + lua_setmetatable(L, -2); + return 1; +} + +static int lupb_symtab_gc(lua_State *L) { + lupb_symtab *lsymtab = luaL_checkudata(L, 1, LUPB_SYMTAB); + upb_symtab_free(lsymtab->symtab); + lsymtab->symtab = NULL; + return 0; +} + +/* TODO(haberman): perhaps this should take a message object instead of a + * serialized string once we have a good story for vending compiled-in + * messages. */ +static int lupb_symtab_add(lua_State *L) { + upb_arena *arena; + size_t i, n, len; + const google_protobuf_FileDescriptorProto *const *files; + google_protobuf_FileDescriptorSet *set; + upb_symtab *s = lupb_symtab_check(L, 1); + const char *str = luaL_checklstring(L, 2, &len); + + lupb_arena_new(L); + arena = lupb_arena_check(L, -1); + + set = google_protobuf_FileDescriptorSet_parse(str, len, arena); + + if (!set) { + luaL_argerror(L, 2, "failed to parse descriptor"); + } + + files = google_protobuf_FileDescriptorSet_file(set, &n); + for (i = 0; i < n; i++) { + CHK(upb_symtab_addfile(s, files[i], &status)); + } + + return 0; +} + +static int lupb_symtab_lookupmsg(lua_State *L) { + const upb_symtab *s = lupb_symtab_check(L, 1); + const upb_msgdef *m = upb_symtab_lookupmsg(s, luaL_checkstring(L, 2)); + lupb_msgdef_pushwrapper(L, m); + return 1; +} + +static int lupb_symtab_lookupenum(lua_State *L) { + const upb_symtab *s = lupb_symtab_check(L, 1); + const upb_enumdef *e = upb_symtab_lookupenum(s, luaL_checkstring(L, 2)); + lupb_enumdef_pushwrapper(L, e); + return 1; +} + +static const struct luaL_Reg lupb_symtab_m[] = { + {"add", lupb_symtab_add}, + {"lookup_msg", lupb_symtab_lookupmsg}, + {"lookup_enum", lupb_symtab_lookupenum}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_symtab_mm[] = { + {"__gc", lupb_symtab_gc}, + {NULL, NULL} +}; + +/* lupb toplevel **************************************************************/ + +static void lupb_setfieldi(lua_State *L, const char *field, int i) { + lua_pushinteger(L, i); + lua_setfield(L, -2, field); +} + +static const struct luaL_Reg lupbdef_toplevel_m[] = { + {"SymbolTable", lupb_symtab_new}, + {NULL, NULL} +}; + +void lupb_def_registertypes(lua_State *L) { + lupb_setfuncs(L, lupbdef_toplevel_m); + + /* Refcounted types. */ + lupb_register_type(L, LUPB_ENUMDEF, lupb_enumdef_m, lupb_enumdef_mm); + lupb_register_type(L, LUPB_FIELDDEF, lupb_fielddef_m, NULL); + lupb_register_type(L, LUPB_FILEDEF, lupb_filedef_m, NULL); + lupb_register_type(L, LUPB_MSGDEF, lupb_msgdef_m, lupb_msgdef_mm); + lupb_register_type(L, LUPB_ONEOFDEF, lupb_oneofdef_m, lupb_oneofdef_mm); + lupb_register_type(L, LUPB_SYMTAB, lupb_symtab_m, lupb_symtab_mm); + + /* Create our object cache. */ + lua_newtable(L); + lua_createtable(L, 0, 1); /* Cache metatable. */ + lua_pushstring(L, "v"); /* Values are weak. */ + lua_setfield(L, -2, "__mode"); + lua_setmetatable(L, -2); + lua_setfield(L, LUA_REGISTRYINDEX, LUPB_OBJCACHE); + + /* Register constants. */ + lupb_setfieldi(L, "LABEL_OPTIONAL", UPB_LABEL_OPTIONAL); + lupb_setfieldi(L, "LABEL_REQUIRED", UPB_LABEL_REQUIRED); + lupb_setfieldi(L, "LABEL_REPEATED", UPB_LABEL_REPEATED); + + lupb_setfieldi(L, "TYPE_DOUBLE", UPB_TYPE_DOUBLE); + lupb_setfieldi(L, "TYPE_FLOAT", UPB_TYPE_FLOAT); + lupb_setfieldi(L, "TYPE_INT64", UPB_TYPE_INT64); + lupb_setfieldi(L, "TYPE_UINT64", UPB_TYPE_UINT64); + lupb_setfieldi(L, "TYPE_INT32", UPB_TYPE_INT32); + lupb_setfieldi(L, "TYPE_BOOL", UPB_TYPE_BOOL); + lupb_setfieldi(L, "TYPE_STRING", UPB_TYPE_STRING); + lupb_setfieldi(L, "TYPE_MESSAGE", UPB_TYPE_MESSAGE); + lupb_setfieldi(L, "TYPE_BYTES", UPB_TYPE_BYTES); + lupb_setfieldi(L, "TYPE_UINT32", UPB_TYPE_UINT32); + lupb_setfieldi(L, "TYPE_ENUM", UPB_TYPE_ENUM); + + lupb_setfieldi(L, "DESCRIPTOR_TYPE_DOUBLE", UPB_DESCRIPTOR_TYPE_DOUBLE); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_FLOAT", UPB_DESCRIPTOR_TYPE_FLOAT); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_INT64", UPB_DESCRIPTOR_TYPE_INT64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_UINT64", UPB_DESCRIPTOR_TYPE_UINT64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_INT32", UPB_DESCRIPTOR_TYPE_INT32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_FIXED64", UPB_DESCRIPTOR_TYPE_FIXED64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_FIXED32", UPB_DESCRIPTOR_TYPE_FIXED32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_BOOL", UPB_DESCRIPTOR_TYPE_BOOL); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_STRING", UPB_DESCRIPTOR_TYPE_STRING); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_GROUP", UPB_DESCRIPTOR_TYPE_GROUP); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_MESSAGE", UPB_DESCRIPTOR_TYPE_MESSAGE); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_BYTES", UPB_DESCRIPTOR_TYPE_BYTES); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_UINT32", UPB_DESCRIPTOR_TYPE_UINT32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_ENUM", UPB_DESCRIPTOR_TYPE_ENUM); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SFIXED32", UPB_DESCRIPTOR_TYPE_SFIXED32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SFIXED64", UPB_DESCRIPTOR_TYPE_SFIXED64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SINT32", UPB_DESCRIPTOR_TYPE_SINT32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SINT64", UPB_DESCRIPTOR_TYPE_SINT64); + + lupb_setfieldi(L, "HANDLER_INT32", UPB_HANDLER_INT32); + lupb_setfieldi(L, "HANDLER_INT64", UPB_HANDLER_INT64); + lupb_setfieldi(L, "HANDLER_UINT32", UPB_HANDLER_UINT32); + lupb_setfieldi(L, "HANDLER_UINT64", UPB_HANDLER_UINT64); + lupb_setfieldi(L, "HANDLER_FLOAT", UPB_HANDLER_FLOAT); + lupb_setfieldi(L, "HANDLER_DOUBLE", UPB_HANDLER_DOUBLE); + lupb_setfieldi(L, "HANDLER_BOOL", UPB_HANDLER_BOOL); + lupb_setfieldi(L, "HANDLER_STARTSTR", UPB_HANDLER_STARTSTR); + lupb_setfieldi(L, "HANDLER_STRING", UPB_HANDLER_STRING); + lupb_setfieldi(L, "HANDLER_ENDSTR", UPB_HANDLER_ENDSTR); + lupb_setfieldi(L, "HANDLER_STARTSUBMSG", UPB_HANDLER_STARTSUBMSG); + lupb_setfieldi(L, "HANDLER_ENDSUBMSG", UPB_HANDLER_ENDSUBMSG); + lupb_setfieldi(L, "HANDLER_STARTSEQ", UPB_HANDLER_STARTSEQ); + lupb_setfieldi(L, "HANDLER_ENDSEQ", UPB_HANDLER_ENDSEQ); + + lupb_setfieldi(L, "SYNTAX_PROTO2", UPB_SYNTAX_PROTO2); + lupb_setfieldi(L, "SYNTAX_PROTO3", UPB_SYNTAX_PROTO3); +} diff --git a/third_party/upb/upb/bindings/lua/msg.c b/third_party/upb/upb/bindings/lua/msg.c new file mode 100644 index 00000000000..5e769b2f0dd --- /dev/null +++ b/third_party/upb/upb/bindings/lua/msg.c @@ -0,0 +1,1060 @@ +/* +** lupb_msg -- Message/Array/Map objects in Lua/C that wrap upb/msg.h +*/ + +#include +#include +#include +#include +#include + +#include "lauxlib.h" +#include "upb/bindings/lua/upb.h" +#include "upb/handlers.h" +#include "upb/legacy_msg_reflection.h" +#include "upb/msg.h" + +#include "upb/port_def.inc" + +/* + * Message/Array/Map objects can be constructed in one of two ways: + * + * 1. To point to existing msg/array/map data inside an arena. + * 2. To create and uniquely own some brand new data. + * + * Case (1) is for when we've parsed some data into an arena (which is faster + * than parsing directly into Lua objects) or when we're pointing at some + * read-only data (like custom options in a def). + * + * Case (2) is for when a user creates the object directly in Lua. + * + * We use the userval of container objects (Message/Array/Map) to store + * references to sub-objects (Strings/Messages/Arrays/Maps). But we need to + * keep the userval in sync with the underlying upb_msg/upb_array/upb_map. + * We populate the userval lazily from the underlying data. + * + * This means that no one may remove/replace any String/Message/Array/Map + * field/entry in the underlying upb_{msg,array,map} behind our back. It's ok + * for entries to be added or for primitives to be modified, but *replacing* + * sub-containers is not. + * + * Luckily parse/merge follow this rule. However clear does not, so it's not + * safe to clear behind our back. + */ + +#define LUPB_ARENA "lupb.arena" + +#define LUPB_MSGCLASS "lupb.msgclass" +#define LUPB_MSGFACTORY "lupb.msgfactory" + +#define LUPB_ARRAY "lupb.array" +#define LUPB_MAP "lupb.map" +#define LUPB_MSG "lupb.msg" +#define LUPB_STRING "lupb.string" + +static int lupb_msg_pushnew(lua_State *L, int narg); + +/* Lazily creates the uservalue if it doesn't exist. */ +static void lupb_getuservalue(lua_State *L, int index) { + lua_getuservalue(L, index); + if (lua_isnil(L, -1)) { + /* Lazily create and set userval. */ + lua_pop(L, 1); /* nil. */ + lua_pushvalue(L, index); /* userdata copy. */ + lua_newtable(L); + lua_setuservalue(L, -2); + lua_pop(L, 1); /* userdata copy. */ + lua_getuservalue(L, index); + } + assert(!lua_isnil(L, -1)); +} + +static void lupb_uservalseti(lua_State *L, int userdata, int index, int val) { + lupb_getuservalue(L, userdata); + lua_pushvalue(L, val); + lua_rawseti(L, -2, index); + lua_pop(L, 1); /* Uservalue. */ +} + +static void lupb_uservalgeti(lua_State *L, int userdata, int index) { + lupb_getuservalue(L, userdata); + lua_rawgeti(L, -1, index); + lua_insert(L, -2); + lua_pop(L, 1); /* Uservalue. */ +} + +/* Pushes a new userdata with the given metatable. */ +static void *lupb_newuserdata(lua_State *L, size_t size, const char *type) { + void *ret = lua_newuserdata(L, size); + + /* Set metatable. */ + luaL_getmetatable(L, type); + UPB_ASSERT(!lua_isnil(L, -1)); /* Should have been created by luaopen_upb. */ + lua_setmetatable(L, -2); + + /* We don't set a uservalue here -- we lazily create it later if necessary. */ + + return ret; +} + + +/* lupb_arena *****************************************************************/ + +/* lupb_arena only exists to wrap a upb_arena. It is never exposed to users; + * it is an internal memory management detail. Other objects refer to this + * object from their userdata to keep the arena-owned data alive. */ + +typedef struct { + upb_arena *arena; +} lupb_arena; + +upb_arena *lupb_arena_check(lua_State *L, int narg) { + lupb_arena *a = luaL_checkudata(L, narg, LUPB_ARENA); + return a ? a->arena : NULL; +} + +int lupb_arena_new(lua_State *L) { + lupb_arena *a = lupb_newuserdata(L, sizeof(lupb_arena), LUPB_ARENA); + + /* TODO(haberman): use Lua alloc func as block allocator? Would need to + * verify that all cases of upb_malloc in msg/table are longjmp-safe. */ + a->arena = upb_arena_new(); + + return 1; +} + +char lupb_arena_cache_key; + +/* Returns the global lupb_arena func that was created in our luaopen(). + * Callers can be guaranteed that it will be alive as long as |L| is. + * TODO(haberman): we shouldn't use a global arena! We should have + * one arena for a parse, or per independently-created message. */ +upb_arena *lupb_arena_get(lua_State *L) { + upb_arena *arena; + + lua_pushlightuserdata(L, &lupb_arena_cache_key); + lua_gettable(L, LUA_REGISTRYINDEX); + arena = lua_touserdata(L, -1); + UPB_ASSERT(arena); + lua_pop(L, 1); + + return arena; +} + +static void lupb_arena_initsingleton(lua_State *L) { + lua_pushlightuserdata(L, &lupb_arena_cache_key); + lupb_arena_new(L); + lua_settable(L, LUA_REGISTRYINDEX); +} + +static int lupb_arena_gc(lua_State *L) { + upb_arena *a = lupb_arena_check(L, 1); + upb_arena_free(a); + return 0; +} + +static const struct luaL_Reg lupb_arena_mm[] = { + {"__gc", lupb_arena_gc}, + {NULL, NULL} +}; + + +/* lupb_msgfactory ************************************************************/ + +/* Userval contains a map of: + * [1] -> SymbolTable (to keep GC-reachable) + * [const upb_msgdef*] -> [lupb_msgclass userdata] + */ + +#define LUPB_MSGFACTORY_SYMTAB 1 + +typedef struct lupb_msgfactory { + upb_msgfactory *factory; +} lupb_msgfactory; + +static int lupb_msgclass_pushnew(lua_State *L, int factory, + const upb_msgdef *md); + +/* lupb_msgfactory helpers. */ + +static lupb_msgfactory *lupb_msgfactory_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_MSGFACTORY); +} + +static void lupb_msgfactory_pushmsgclass(lua_State *L, int narg, + const upb_msgdef *md) { + lupb_getuservalue(L, narg); + lua_pushlightuserdata(L, (void*)md); + lua_rawget(L, -2); + + if (lua_isnil(L, -1)) { + lua_pop(L, 1); + /* TODO: verify md is in symtab? */ + lupb_msgclass_pushnew(L, narg, md); + + /* Set in userval. */ + lua_pushlightuserdata(L, (void*)md); + lua_pushvalue(L, -2); + lua_rawset(L, -4); + } +} + +static int lupb_msgfactory_gc(lua_State *L) { + lupb_msgfactory *lfactory = lupb_msgfactory_check(L, 1); + + if (lfactory->factory) { + upb_msgfactory_free(lfactory->factory); + lfactory->factory = NULL; + } + + return 0; +} + +/* lupb_msgfactory Public API. */ + +/** + * lupb_msgfactory_new() + * + * Handles: + * msgfactory = upb.MessageFactory(symtab) + * + * Creates a new, empty MessageFactory for the given SymbolTable. + * Message classes will be created on demand when the user calls + * msgfactory.get_message_class(). + */ +static int lupb_msgfactory_new(lua_State *L) { + const upb_symtab *symtab = lupb_symtab_check(L, 1); + + lupb_msgfactory *lmsgfactory = + lupb_newuserdata(L, sizeof(lupb_msgfactory), LUPB_MSGFACTORY); + lmsgfactory->factory = upb_msgfactory_new(symtab); + lupb_uservalseti(L, -1, LUPB_MSGFACTORY_SYMTAB, 1); + + return 1; +} + +/** + * lupb_msgfactory_getmsgclass() + * + * Handles: + * MessageClass = factory.get_message_class(message_name) + */ +static int lupb_msgfactory_getmsgclass(lua_State *L) { + lupb_msgfactory *lfactory = lupb_msgfactory_check(L, 1); + const upb_symtab *symtab = upb_msgfactory_symtab(lfactory->factory); + const upb_msgdef *m = upb_symtab_lookupmsg(symtab, luaL_checkstring(L, 2)); + + if (!m) { + luaL_error(L, "No such message type: %s\n", lua_tostring(L, 2)); + } + + lupb_msgfactory_pushmsgclass(L, 1, m); + + return 1; +} + +static const struct luaL_Reg lupb_msgfactory_m[] = { + {"get_message_class", lupb_msgfactory_getmsgclass}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_msgfactory_mm[] = { + {"__gc", lupb_msgfactory_gc}, + {NULL, NULL} +}; + + +/* lupb_msgclass **************************************************************/ + +/* Userval contains a map of: + * [1] -> MessageFactory (to keep GC-reachable) + * [const upb_msgdef*] -> [lupb_msgclass userdata] + */ + +#define LUPB_MSGCLASS_FACTORY 1 + +struct lupb_msgclass { + const upb_msglayout *layout; + const upb_msgdef *msgdef; + const lupb_msgfactory *lfactory; +}; + +/* Type-checks for assigning to a message field. */ +static upb_msgval lupb_array_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f); +static upb_msgval lupb_map_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f); +static const lupb_msgclass *lupb_msg_getsubmsgclass(lua_State *L, int narg, + const upb_fielddef *f); +static const lupb_msgclass *lupb_msg_msgclassfor(lua_State *L, int narg, + const upb_msgdef *md); + +const lupb_msgclass *lupb_msgclass_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_MSGCLASS); +} + +const upb_msglayout *lupb_msgclass_getlayout(lua_State *L, int narg) { + return lupb_msgclass_check(L, narg)->layout; +} + +const upb_msgdef *lupb_msgclass_getmsgdef(const lupb_msgclass *lmsgclass) { + return lmsgclass->msgdef; +} + +upb_msgfactory *lupb_msgclass_getfactory(const lupb_msgclass *lmsgclass) { + return lmsgclass->lfactory->factory; +} + +/** + * lupb_msgclass_typecheck() + * + * Verifies that the expected msgclass matches the actual. If not, raises a Lua + * error. + */ +static void lupb_msgclass_typecheck(lua_State *L, const lupb_msgclass *expected, + const lupb_msgclass *actual) { + if (expected != actual) { + luaL_error(L, "Message had incorrect type, expected '%s', got '%s'", + upb_msgdef_fullname(expected->msgdef), + upb_msgdef_fullname(actual->msgdef)); + } +} + +static const lupb_msgclass *lupb_msgclass_msgclassfor(lua_State *L, int narg, + const upb_msgdef *md) { + lupb_uservalgeti(L, narg, LUPB_MSGCLASS_FACTORY); + lupb_msgfactory_pushmsgclass(L, -1, md); + return lupb_msgclass_check(L, -1); +} + +/** + * lupb_msgclass_getsubmsgclass() + * + * Given a MessageClass at index |narg| and the submessage field |f|, returns + * the message class for this field. + * + * Currently we do a hash table lookup for this. If we wanted we could try to + * optimize this by caching these pointers in our msgclass, in an array indexed + * by field index. We would still need to fall back to calling msgclassfor(), + * unless we wanted to eagerly create message classes for all submessages. But + * for big schemas that might be a lot of things to build, and we might end up + * not using most of them. */ +static const lupb_msgclass *lupb_msgclass_getsubmsgclass(lua_State *L, int narg, + const upb_fielddef *f) { + if (upb_fielddef_type(f) != UPB_TYPE_MESSAGE) { + return NULL; + } + + return lupb_msgclass_msgclassfor(L, narg, upb_fielddef_msgsubdef(f)); +} + +static int lupb_msgclass_pushnew(lua_State *L, int factory, + const upb_msgdef *md) { + const lupb_msgfactory *lfactory = lupb_msgfactory_check(L, factory); + lupb_msgclass *lmc = lupb_newuserdata(L, sizeof(*lmc), LUPB_MSGCLASS); + + lupb_uservalseti(L, -1, LUPB_MSGCLASS_FACTORY, factory); + lmc->layout = upb_msgfactory_getlayout(lfactory->factory, md); + lmc->lfactory = lfactory; + lmc->msgdef = md; + + return 1; +} + +/* MessageClass Public API. */ + +/** + * lupb_msgclass_call() + * + * Handles: + * msg = MessageClass() + * + * Creates a new message from the given MessageClass. + */ +static int lupb_msgclass_call(lua_State *L) { + lupb_msg_pushnew(L, 1); + return 1; +} + +static const struct luaL_Reg lupb_msgclass_mm[] = { + {"__call", lupb_msgclass_call}, + {NULL, NULL} +}; + + +/* upb <-> Lua type conversion ************************************************/ + +static bool lupb_istypewrapped(upb_fieldtype_t type) { + return type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES || + type == UPB_TYPE_MESSAGE; +} + +static upb_msgval lupb_tomsgval(lua_State *L, upb_fieldtype_t type, int narg, + const lupb_msgclass *lmsgclass) { + switch (type) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: + return upb_msgval_int32(lupb_checkint32(L, narg)); + case UPB_TYPE_INT64: + return upb_msgval_int64(lupb_checkint64(L, narg)); + case UPB_TYPE_UINT32: + return upb_msgval_uint32(lupb_checkuint32(L, narg)); + case UPB_TYPE_UINT64: + return upb_msgval_uint64(lupb_checkuint64(L, narg)); + case UPB_TYPE_DOUBLE: + return upb_msgval_double(lupb_checkdouble(L, narg)); + case UPB_TYPE_FLOAT: + return upb_msgval_float(lupb_checkfloat(L, narg)); + case UPB_TYPE_BOOL: + return upb_msgval_bool(lupb_checkbool(L, narg)); + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + size_t len; + const char *ptr = lupb_checkstring(L, narg, &len); + return upb_msgval_makestr(ptr, len); + } + case UPB_TYPE_MESSAGE: + UPB_ASSERT(lmsgclass); + return upb_msgval_msg(lupb_msg_checkmsg(L, narg, lmsgclass)); + } + UPB_UNREACHABLE(); +} + +static void lupb_pushmsgval(lua_State *L, upb_fieldtype_t type, + upb_msgval val) { + switch (type) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: + lupb_pushint32(L, upb_msgval_getint32(val)); + return; + case UPB_TYPE_INT64: + lupb_pushint64(L, upb_msgval_getint64(val)); + return; + case UPB_TYPE_UINT32: + lupb_pushuint32(L, upb_msgval_getuint32(val)); + return; + case UPB_TYPE_UINT64: + lupb_pushuint64(L, upb_msgval_getuint64(val)); + return; + case UPB_TYPE_DOUBLE: + lupb_pushdouble(L, upb_msgval_getdouble(val)); + return; + case UPB_TYPE_FLOAT: + lupb_pushfloat(L, upb_msgval_getfloat(val)); + return; + case UPB_TYPE_BOOL: + lua_pushboolean(L, upb_msgval_getbool(val)); + return; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + case UPB_TYPE_MESSAGE: + break; /* Shouldn't call this function. */ + } + UPB_UNREACHABLE(); +} + + +/* lupb_array *****************************************************************/ + +/* A strongly typed array. Implemented by wrapping upb_array. + * + * - we only allow integer indices. + * - all entries must have the correct type. + * - we do not allow "holes" in the array; you can only assign to an existing + * index or one past the end (which will grow the array by one). + * + * For string/submessage entries we keep in the userval: + * + * [number index] -> [lupb_string/lupb_msg userdata] + */ + +typedef struct { + /* Only needed for array of message. This wastes space in the non-message + * case but simplifies the code. Could optimize away if desired. */ + const lupb_msgclass *lmsgclass; + upb_array *arr; + upb_fieldtype_t type; +} lupb_array; + +#define ARRAY_MSGCLASS_INDEX 0 + +static lupb_array *lupb_array_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_ARRAY); +} + +/** + * lupb_array_typecheck() + * + * Verifies that the lupb_array object at index |narg| can be safely assigned + * to the field |f| of the lupb_msg object at index |msg|. If this is safe, + * returns a upb_msgval representing the array. Otherwise, throws a Lua error. + */ +static upb_msgval lupb_array_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f) { + lupb_array *larray = lupb_array_check(L, narg); + + if (upb_array_type(larray->arr) != upb_fielddef_type(f) || + lupb_msg_getsubmsgclass(L, msg, f) != larray->lmsgclass) { + luaL_error(L, "Array had incorrect type (expected: %d, got: %d)", + (int)upb_fielddef_type(f), (int)upb_array_type(larray->arr)); + } + + if (upb_array_type(larray->arr) == UPB_TYPE_MESSAGE) { + lupb_msgclass_typecheck(L, lupb_msg_getsubmsgclass(L, msg, f), + larray->lmsgclass); + } + + return upb_msgval_arr(larray->arr); +} + +/** + * lupb_array_checkindex() + * + * Checks the array index at Lua stack index |narg| to verify that it is an + * integer between 1 and |max|, inclusively. Also corrects it to be zero-based + * for C. + * + * We use "int" because of lua_rawseti/lua_rawgeti -- can re-evaluate if we want + * arrays bigger than 2^31. + */ +static int lupb_array_checkindex(lua_State *L, int narg, uint32_t max) { + uint32_t n = lupb_checkuint32(L, narg); + if (n == 0 || n > max || n > INT_MAX) { + luaL_error(L, "Invalid array index: expected between 1 and %d", (int)max); + } + return n - 1; /* Lua uses 1-based indexing. :( */ +} + +/* lupb_array Public API */ + +static int lupb_array_new(lua_State *L) { + lupb_array *larray; + upb_fieldtype_t type; + const lupb_msgclass *lmsgclass = NULL; + + if (lua_type(L, 1) == LUA_TNUMBER) { + type = lupb_checkfieldtype(L, 1); + } else { + type = UPB_TYPE_MESSAGE; + lmsgclass = lupb_msgclass_check(L, 1); + lupb_uservalseti(L, -1, ARRAY_MSGCLASS_INDEX, 1); /* GC-root lmsgclass. */ + } + + larray = lupb_newuserdata(L, sizeof(*larray), LUPB_ARRAY); + larray->type = type; + larray->lmsgclass = lmsgclass; + larray->arr = upb_array_new(lupb_arena_get(L)); + + return 1; +} + +static int lupb_array_newindex(lua_State *L) { + lupb_array *larray = lupb_array_check(L, 1); + upb_fieldtype_t type = upb_array_type(larray->arr); + uint32_t n = lupb_array_checkindex(L, 2, upb_array_size(larray->arr) + 1); + upb_msgval msgval = lupb_tomsgval(L, type, 3, larray->lmsgclass); + + upb_array_set(larray->arr, larray->type, n, msgval, lupb_arena_get(L)); + + if (lupb_istypewrapped(type)) { + lupb_uservalseti(L, 1, n, 3); + } + + return 0; /* 1 for chained assignments? */ +} + +static int lupb_array_index(lua_State *L) { + lupb_array *larray = lupb_array_check(L, 1); + upb_array *array = larray->arr; + uint32_t n = lupb_array_checkindex(L, 2, upb_array_size(array)); + upb_fieldtype_t type = upb_array_type(array); + + if (lupb_istypewrapped(type)) { + lupb_uservalgeti(L, 1, n); + } else { + lupb_pushmsgval(L, upb_array_type(array), + upb_array_get(array, larray->type, n)); + } + + return 1; +} + +static int lupb_array_len(lua_State *L) { + lupb_array *larray = lupb_array_check(L, 1); + lua_pushnumber(L, upb_array_size(larray->arr)); + return 1; +} + +static const struct luaL_Reg lupb_array_mm[] = { + {"__index", lupb_array_index}, + {"__len", lupb_array_len}, + {"__newindex", lupb_array_newindex}, + {NULL, NULL} +}; + + +/* lupb_map *******************************************************************/ + +/* A map object. Implemented by wrapping upb_map. + * + * When the value type is string/bytes/message, the userval consists of: + * + * [Lua number/string] -> [lupb_string/lupb_msg userdata] + * + * For other value types we don't use the userdata. + */ + +typedef struct { + const lupb_msgclass *value_lmsgclass; + upb_map *map; +} lupb_map; + +#define MAP_MSGCLASS_INDEX 0 + +/* lupb_map internal functions */ + +static lupb_map *lupb_map_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_ARRAY); +} + +/** + * lupb_map_typecheck() + * + * Checks that the lupb_map at index |narg| can be safely assigned to the + * field |f| of the message at index |msg|. If so, returns a upb_msgval for + * this map. Otherwise, raises a Lua error. + */ +static upb_msgval lupb_map_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f) { + lupb_map *lmap = lupb_map_check(L, narg); + upb_map *map = lmap->map; + const upb_msgdef *entry = upb_fielddef_msgsubdef(f); + const upb_fielddef *key_field = upb_msgdef_itof(entry, UPB_MAPENTRY_KEY); + const upb_fielddef *value_field = upb_msgdef_itof(entry, UPB_MAPENTRY_VALUE); + + UPB_ASSERT(entry && key_field && value_field); + + if (upb_map_keytype(map) != upb_fielddef_type(key_field)) { + luaL_error(L, "Map key type invalid"); + } + + if (upb_map_valuetype(map) != upb_fielddef_type(value_field)) { + luaL_error(L, "Map had incorrect value type (expected: %s, got: %s)", + upb_fielddef_type(value_field), upb_map_valuetype(map)); + } + + if (upb_map_valuetype(map) == UPB_TYPE_MESSAGE) { + lupb_msgclass_typecheck( + L, lupb_msg_msgclassfor(L, msg, upb_fielddef_msgsubdef(value_field)), + lmap->value_lmsgclass); + } + + return upb_msgval_map(map); +} + +/* lupb_map Public API */ + +/** + * lupb_map_new + * + * Handles: + * new_map = upb.Map(key_type, value_type) + */ +static int lupb_map_new(lua_State *L) { + lupb_map *lmap; + upb_fieldtype_t key_type = lupb_checkfieldtype(L, 1); + upb_fieldtype_t value_type; + const lupb_msgclass *value_lmsgclass = NULL; + + if (lua_type(L, 2) == LUA_TNUMBER) { + value_type = lupb_checkfieldtype(L, 2); + } else { + value_type = UPB_TYPE_MESSAGE; + } + + lmap = lupb_newuserdata(L, sizeof(*lmap), LUPB_MAP); + + if (value_type == UPB_TYPE_MESSAGE) { + value_lmsgclass = lupb_msgclass_check(L, 2); + lupb_uservalseti(L, -1, MAP_MSGCLASS_INDEX, 2); /* GC-root lmsgclass. */ + } + + lmap->value_lmsgclass = value_lmsgclass; + lmap->map = upb_map_new(key_type, value_type, lupb_arena_get(L)); + + return 1; +} + +/** + * lupb_map_index + * + * Handles: + * map[key] + */ +static int lupb_map_index(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + upb_map *map = lmap->map; + upb_fieldtype_t valtype = upb_map_valuetype(map); + /* We don't always use "key", but this call checks the key type. */ + upb_msgval key = lupb_tomsgval(L, upb_map_keytype(map), 2, NULL); + + if (lupb_istypewrapped(valtype)) { + /* Userval contains the full map, lookup there by key. */ + lupb_getuservalue(L, 1); + lua_pushvalue(L, 2); + lua_rawget(L, -2); + + if (lua_isnil(L, -1)) { + /* TODO: lazy read from upb_map */ + } + } else { + /* Lookup in upb_map. */ + upb_msgval val; + if (upb_map_get(map, key, &val)) { + lupb_pushmsgval(L, upb_map_valuetype(map), val); + } else { + lua_pushnil(L); + } + } + + return 1; +} + +/** + * lupb_map_len + * + * Handles: + * map_len = #map + */ +static int lupb_map_len(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + lua_pushnumber(L, upb_map_size(lmap->map)); + return 1; +} + +/** + * lupb_map_newindex + * + * Handles: + * map[key] = val + * map[key] = nil # to remove from map + */ +static int lupb_map_newindex(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + upb_map *map = lmap->map; + upb_msgval key = lupb_tomsgval(L, upb_map_keytype(map), 2, NULL); + + if (lua_isnil(L, 3)) { + /* Delete from map. */ + upb_map_del(map, key); + + if (lupb_istypewrapped(upb_map_valuetype(map))) { + /* Delete in userval. */ + lupb_getuservalue(L, 1); + lua_pushvalue(L, 2); + lua_pushnil(L); + lua_rawset(L, -3); + lua_pop(L, 1); + } + } else { + /* Set in map. */ + upb_msgval val = + lupb_tomsgval(L, upb_map_valuetype(map), 3, lmap->value_lmsgclass); + + upb_map_set(map, key, val, NULL); + + if (lupb_istypewrapped(upb_map_valuetype(map))) { + /* Set in userval. */ + lupb_getuservalue(L, 1); + lua_pushvalue(L, 2); + lua_pushvalue(L, 3); + lua_rawset(L, -3); + lua_pop(L, 1); + } + } + + return 0; +} + +/* upb_mapiter [[[ */ + +static int lupb_mapiter_next(lua_State *L) { + upb_mapiter *i = lua_touserdata(L, lua_upvalueindex(1)); + lupb_map *lmap = lupb_map_check(L, 1); + upb_map *map = lmap->map; + + if (upb_mapiter_done(i)) { + return 0; + } + + lupb_pushmsgval(L, upb_map_keytype(map), upb_mapiter_key(i)); + lupb_pushmsgval(L, upb_map_valuetype(map), upb_mapiter_value(i)); + upb_mapiter_next(i); + + return 2; +} + +static int lupb_map_pairs(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + + if (lupb_istypewrapped(upb_map_keytype(lmap->map)) || + lupb_istypewrapped(upb_map_valuetype(lmap->map))) { + /* Complex key or value type. + * Sync upb_map to userval if necessary, then iterate over userval. */ + + /* TODO: Lua tables don't know how many entries they have, gah!. */ + return 1; + } else { + /* Simple key and value type, iterate over the upb_map directly. */ + upb_mapiter *i = lua_newuserdata(L, upb_mapiter_sizeof()); + + upb_mapiter_begin(i, lmap->map); + lua_pushvalue(L, 1); + + /* Upvalues are [upb_mapiter, lupb_map]. */ + lua_pushcclosure(L, &lupb_mapiter_next, 2); + + return 1; + } +} + +/* upb_mapiter ]]] */ + +static const struct luaL_Reg lupb_map_mm[] = { + {"__index", lupb_map_index}, + {"__len", lupb_map_len}, + {"__newindex", lupb_map_newindex}, + {"__pairs", lupb_map_pairs}, + {NULL, NULL} +}; + + +/* lupb_msg *******************************************************************/ + +/* A message object. Implemented by wrapping upb_msg. + * + * Our userval contains: + * + * - [0] -> our message class + * - [lupb_fieldindex(f)] -> [lupb_{string,array,map,msg} userdata] + * + * Fields with scalar number/bool types don't go in the userval. + */ + +#define LUPB_MSG_MSGCLASSINDEX 0 +#define LUPB_MSG_ARENA -1 + +int lupb_fieldindex(const upb_fielddef *f) { + return upb_fielddef_index(f) + 1; /* 1-based Lua arrays. */ +} + + +typedef struct { + const lupb_msgclass *lmsgclass; + upb_msg *msg; +} lupb_msg; + +/* lupb_msg helpers */ + +static bool in_userval(const upb_fielddef *f) { + return lupb_istypewrapped(upb_fielddef_type(f)) || upb_fielddef_isseq(f) || + upb_fielddef_ismap(f); +} + +lupb_msg *lupb_msg_check(lua_State *L, int narg) { + lupb_msg *msg = luaL_checkudata(L, narg, LUPB_MSG); + if (!msg->lmsgclass) luaL_error(L, "called into dead msg"); + return msg; +} + +const upb_msg *lupb_msg_checkmsg(lua_State *L, int narg, + const lupb_msgclass *lmsgclass) { + lupb_msg *lmsg = lupb_msg_check(L, narg); + lupb_msgclass_typecheck(L, lmsgclass, lmsg->lmsgclass); + return lmsg->msg; +} + +upb_msg *lupb_msg_checkmsg2(lua_State *L, int narg, + const upb_msglayout **layout) { + lupb_msg *lmsg = lupb_msg_check(L, narg); + *layout = lmsg->lmsgclass->layout; + return lmsg->msg; +} + +const upb_msgdef *lupb_msg_checkdef(lua_State *L, int narg) { + return lupb_msg_check(L, narg)->lmsgclass->msgdef; +} + +static const upb_fielddef *lupb_msg_checkfield(lua_State *L, + const lupb_msg *msg, + int fieldarg) { + size_t len; + const char *fieldname = luaL_checklstring(L, fieldarg, &len); + const upb_msgdef *msgdef = msg->lmsgclass->msgdef; + const upb_fielddef *f = upb_msgdef_ntof(msgdef, fieldname, len); + + if (!f) { + const char *msg = lua_pushfstring(L, "no such field: %s", fieldname); + luaL_argerror(L, fieldarg, msg); + return NULL; /* Never reached. */ + } + + return f; +} + +static const lupb_msgclass *lupb_msg_msgclassfor(lua_State *L, int narg, + const upb_msgdef *md) { + lupb_uservalgeti(L, narg, LUPB_MSG_MSGCLASSINDEX); + return lupb_msgclass_msgclassfor(L, -1, md); +} + +static const lupb_msgclass *lupb_msg_getsubmsgclass(lua_State *L, int narg, + const upb_fielddef *f) { + lupb_uservalgeti(L, narg, LUPB_MSG_MSGCLASSINDEX); + return lupb_msgclass_getsubmsgclass(L, -1, f); +} + +int lupb_msg_pushref(lua_State *L, int msgclass, upb_msg *msg) { + const lupb_msgclass *lmsgclass = lupb_msgclass_check(L, msgclass); + lupb_msg *lmsg = lupb_newuserdata(L, sizeof(lupb_msg), LUPB_MSG); + + lmsg->lmsgclass = lmsgclass; + lmsg->msg = msg; + + lupb_uservalseti(L, -1, LUPB_MSG_MSGCLASSINDEX, msgclass); + lupb_uservalseti(L, -1, LUPB_MSG_ARENA, -2); + + return 1; +} + +/* lupb_msg Public API */ + +/** + * lupb_msg_pushnew + * + * Handles: + * new_msg = MessageClass() + */ +static int lupb_msg_pushnew(lua_State *L, int narg) { + const lupb_msgclass *lmsgclass = lupb_msgclass_check(L, narg); + lupb_msg *lmsg = lupb_newuserdata(L, sizeof(lupb_msg), LUPB_MSG); + + lmsg->lmsgclass = lmsgclass; + lmsg->msg = upb_msg_new(lmsgclass->layout, lupb_arena_get(L)); + + lupb_uservalseti(L, -1, LUPB_MSG_MSGCLASSINDEX, narg); + + return 1; +} + +/** + * lupb_msg_index + * + * Handles: + * msg.foo + * msg["foo"] + * msg[field_descriptor] # (for extensions) (TODO) + */ +static int lupb_msg_index(lua_State *L) { + lupb_msg *lmsg = lupb_msg_check(L, 1); + const upb_fielddef *f = lupb_msg_checkfield(L, lmsg, 2); + const upb_msglayout *l = lmsg->lmsgclass->layout; + int field_index = upb_fielddef_index(f); + + if (in_userval(f)) { + lupb_uservalgeti(L, 1, lupb_fieldindex(f)); + + if (lua_isnil(L, -1)) { + /* Check if we need to lazily create wrapper. */ + if (upb_fielddef_isseq(f)) { + /* TODO(haberman) */ + } else if (upb_fielddef_issubmsg(f)) { + /* TODO(haberman) */ + } else { + UPB_ASSERT(upb_fielddef_isstring(f)); + if (upb_msg_has(lmsg->msg, field_index, l)) { + upb_msgval val = upb_msg_get(lmsg->msg, field_index, l); + lua_pop(L, 1); + lua_pushlstring(L, val.str.data, val.str.size); + lupb_uservalseti(L, 1, lupb_fieldindex(f), -1); + } + } + } + } else { + upb_msgval val = upb_msg_get(lmsg->msg, field_index, l); + lupb_pushmsgval(L, upb_fielddef_type(f), val); + } + + return 1; +} + +/** + * lupb_msg_newindex() + * + * Handles: + * msg.foo = bar + * msg["foo"] = bar + * msg[field_descriptor] = bar # (for extensions) (TODO) + */ +static int lupb_msg_newindex(lua_State *L) { + lupb_msg *lmsg = lupb_msg_check(L, 1); + const upb_fielddef *f = lupb_msg_checkfield(L, lmsg, 2); + upb_fieldtype_t type = upb_fielddef_type(f); + int field_index = upb_fielddef_index(f); + upb_msgval msgval; + + /* Typecheck and get msgval. */ + + if (upb_fielddef_isseq(f)) { + msgval = lupb_array_typecheck(L, 3, 1, f); + } else if (upb_fielddef_ismap(f)) { + msgval = lupb_map_typecheck(L, 3, 1, f); + } else { + const lupb_msgclass *lmsgclass = NULL; + + if (type == UPB_TYPE_MESSAGE) { + lmsgclass = lupb_msg_getsubmsgclass(L, 1, f); + } + + msgval = lupb_tomsgval(L, type, 3, lmsgclass); + } + + /* Set in upb_msg and userval (if necessary). */ + + upb_msg_set(lmsg->msg, field_index, msgval, lmsg->lmsgclass->layout); + + if (in_userval(f)) { + lupb_uservalseti(L, 1, lupb_fieldindex(f), 3); + } + + return 0; /* 1 for chained assignments? */ +} + +static const struct luaL_Reg lupb_msg_mm[] = { + {"__index", lupb_msg_index}, + {"__newindex", lupb_msg_newindex}, + {NULL, NULL} +}; + + +/* lupb_msg toplevel **********************************************************/ + +static const struct luaL_Reg lupb_msg_toplevel_m[] = { + {"Array", lupb_array_new}, + {"Map", lupb_map_new}, + {"MessageFactory", lupb_msgfactory_new}, + {NULL, NULL} +}; + +void lupb_msg_registertypes(lua_State *L) { + lupb_setfuncs(L, lupb_msg_toplevel_m); + + lupb_register_type(L, LUPB_ARENA, NULL, lupb_arena_mm); + lupb_register_type(L, LUPB_MSGCLASS, NULL, lupb_msgclass_mm); + lupb_register_type(L, LUPB_MSGFACTORY, lupb_msgfactory_m, lupb_msgfactory_mm); + lupb_register_type(L, LUPB_ARRAY, NULL, lupb_array_mm); + lupb_register_type(L, LUPB_MAP, NULL, lupb_map_mm); + lupb_register_type(L, LUPB_MSG, NULL, lupb_msg_mm); + + lupb_arena_initsingleton(L); +} diff --git a/third_party/upb/upb/bindings/lua/upb.c b/third_party/upb/upb/bindings/lua/upb.c new file mode 100644 index 00000000000..38fd24a27ef --- /dev/null +++ b/third_party/upb/upb/bindings/lua/upb.c @@ -0,0 +1,245 @@ +/* +** require("lua") -- A Lua extension for upb. +** +** Exposes only the core library +** (sub-libraries are exposed in other extensions). +** +** 64-bit woes: Lua can only represent numbers of type lua_Number (which is +** double unless the user specifically overrides this). Doubles can represent +** the entire range of 64-bit integers, but lose precision once the integers are +** greater than 2^53. +** +** Lua 5.3 is adding support for integers, which will allow for 64-bit +** integers (which can be interpreted as signed or unsigned). +** +** LuaJIT supports 64-bit signed and unsigned boxed representations +** through its "cdata" mechanism, but this is not portable to regular Lua. +** +** Hopefully Lua 5.3 will come soon enough that we can either use Lua 5.3 +** integer support or LuaJIT 64-bit cdata for users that need the entire +** domain of [u]int64 values. +*/ + +#include +#include +#include +#include +#include "lauxlib.h" +#include "upb/bindings/lua/upb.h" +#include "upb/handlers.h" +#include "upb/msg.h" + + +/* Lua compatibility code *****************************************************/ + +/* Lua 5.1 and Lua 5.2 have slightly incompatible APIs. A little bit of + * compatibility code can help hide the difference. Not too many people still + * use Lua 5.1 but LuaJIT uses the Lua 5.1 API in some ways. */ + +#if LUA_VERSION_NUM == 501 + +/* taken from lua 5.2's source. */ +void *luaL_testudata(lua_State *L, int ud, const char *tname) { + void *p = lua_touserdata(L, ud); + if (p != NULL) { /* value is a userdata? */ + if (lua_getmetatable(L, ud)) { /* does it have a metatable? */ + luaL_getmetatable(L, tname); /* get correct metatable */ + if (!lua_rawequal(L, -1, -2)) /* not the same? */ + p = NULL; /* value is a userdata with wrong metatable */ + lua_pop(L, 2); /* remove both metatables */ + return p; + } + } + return NULL; /* value is not a userdata with a metatable */ +} + +static void lupb_newlib(lua_State *L, const char *name, const luaL_Reg *funcs) { + luaL_register(L, name, funcs); +} + +#elif LUA_VERSION_NUM == 502 + +int luaL_typerror(lua_State *L, int narg, const char *tname) { + const char *msg = lua_pushfstring(L, "%s expected, got %s", + tname, luaL_typename(L, narg)); + return luaL_argerror(L, narg, msg); +} + +static void lupb_newlib(lua_State *L, const char *name, const luaL_Reg *funcs) { + /* Lua 5.2 modules are not expected to set a global variable, so "name" is + * unused. */ + UPB_UNUSED(name); + + /* Can't use luaL_newlib(), because funcs is not the actual array. + * Could (micro-)optimize this a bit to count funcs for initial table size. */ + lua_createtable(L, 0, 8); + luaL_setfuncs(L, funcs, 0); +} + +#else +#error Only Lua 5.1 and 5.2 are supported +#endif + +/* Shims for upcoming Lua 5.3 functionality. */ +bool lua_isinteger(lua_State *L, int argn) { + UPB_UNUSED(L); + UPB_UNUSED(argn); + return false; +} + + +/* Utility functions **********************************************************/ + +/* We store our module table in the registry, keyed by ptr. + * For more info about the motivation/rationale, see this thread: + * http://thread.gmane.org/gmane.comp.lang.lua.general/110632 */ +bool lupb_openlib(lua_State *L, void *ptr, const char *name, + const luaL_Reg *funcs) { + /* Lookup cached module table. */ + lua_pushlightuserdata(L, ptr); + lua_rawget(L, LUA_REGISTRYINDEX); + if (!lua_isnil(L, -1)) { + return true; + } + + lupb_newlib(L, name, funcs); + + /* Save module table in cache. */ + lua_pushlightuserdata(L, ptr); + lua_pushvalue(L, -2); + lua_rawset(L, LUA_REGISTRYINDEX); + + return false; +} + +void lupb_checkstatus(lua_State *L, upb_status *s) { + if (!upb_ok(s)) { + lua_pushstring(L, upb_status_errmsg(s)); + lua_error(L); + } +} + +/* Scalar type mapping ********************************************************/ + +/* Functions that convert scalar/primitive values (numbers, strings, bool) + * between Lua and C/upb. Handles type/range checking. */ + +bool lupb_checkbool(lua_State *L, int narg) { + if (!lua_isboolean(L, narg)) { + luaL_error(L, "must be true or false"); + } + return lua_toboolean(L, narg); +} + +/* Unlike luaL_checkstring(), this does not allow implicit conversion to + * string. */ +const char *lupb_checkstring(lua_State *L, int narg, size_t *len) { + if (lua_type(L, narg) != LUA_TSTRING) { + luaL_error(L, "Expected string"); + } + + return lua_tolstring(L, narg, len); +} + +/* Unlike luaL_checkinteger, these do not implicitly convert from string or + * round an existing double value. We allow floating-point input, but only if + * the actual value is integral. */ +#define INTCHECK(type, ctype) \ + ctype lupb_check##type(lua_State *L, int narg) { \ + double n; \ + ctype i; \ + if (lua_isinteger(L, narg)) { \ + return lua_tointeger(L, narg); \ + } \ + \ + /* Prevent implicit conversion from string. */ \ + luaL_checktype(L, narg, LUA_TNUMBER); \ + n = lua_tonumber(L, narg); \ + \ + i = (ctype)n; \ + if ((double)i != n) { \ + /* double -> ctype truncated or rounded. */ \ + luaL_error(L, "number %f was not an integer or out of range for " #type, \ + n); \ + } \ + return i; \ + } \ + void lupb_push##type(lua_State *L, ctype val) { \ + /* TODO: push integer for Lua >= 5.3, 64-bit cdata for LuaJIT. */ \ + /* This is lossy for some [u]int64 values, which isn't great, but */ \ + /* crashing when we encounter these values seems worse. */ \ + lua_pushnumber(L, val); \ + } + +INTCHECK(int64, int64_t) +INTCHECK(int32, int32_t) +INTCHECK(uint64, uint64_t) +INTCHECK(uint32, uint32_t) + +double lupb_checkdouble(lua_State *L, int narg) { + /* If we were being really hard-nosed here, we'd check whether the input was + * an integer that has no precise double representation. But doubles aren't + * generally expected to be exact like integers are, and worse this could + * cause data-dependent runtime errors: one run of the program could work fine + * because the integer calculations happened to be exactly representable in + * double, while the next could crash because of subtly different input. */ + + luaL_checktype(L, narg, LUA_TNUMBER); /* lua_tonumber() auto-converts. */ + return lua_tonumber(L, narg); +} + +float lupb_checkfloat(lua_State *L, int narg) { + /* We don't worry about checking whether the input can be exactly converted to + * float -- see above. */ + + luaL_checktype(L, narg, LUA_TNUMBER); /* lua_tonumber() auto-converts. */ + return lua_tonumber(L, narg); +} + +void lupb_pushdouble(lua_State *L, double d) { + lua_pushnumber(L, d); +} + +void lupb_pushfloat(lua_State *L, float d) { + lua_pushnumber(L, d); +} + + +static const struct luaL_Reg lupb_toplevel_m[] = { + {NULL, NULL} +}; + +void lupb_register_type(lua_State *L, const char *name, const luaL_Reg *m, + const luaL_Reg *mm) { + luaL_newmetatable(L, name); + + if (mm) { + lupb_setfuncs(L, mm); + } + + if (m) { + /* Methods go in the mt's __index method. This implies that you can' + * implement __index and also have methods. */ + lua_getfield(L, -1, "__index"); + lupb_assert(L, lua_isnil(L, -1)); + lua_pop(L, 1); + + lua_createtable(L, 0, 0); + lupb_setfuncs(L, m); + lua_setfield(L, -2, "__index"); + } + + lua_pop(L, 1); /* The mt. */ +} + +int luaopen_upb_c(lua_State *L) { + static char module_key; + if (lupb_openlib(L, &module_key, "upb_c", lupb_toplevel_m)) { + return 1; + } + + lupb_def_registertypes(L); + lupb_msg_registertypes(L); + + return 1; /* Return package table. */ +} diff --git a/third_party/upb/upb/bindings/lua/upb.h b/third_party/upb/upb/bindings/lua/upb.h new file mode 100644 index 00000000000..51d8acf9e45 --- /dev/null +++ b/third_party/upb/upb/bindings/lua/upb.h @@ -0,0 +1,127 @@ +/* +** Shared definitions for upb Lua modules. +*/ + +#ifndef UPB_LUA_UPB_H_ +#define UPB_LUA_UPB_H_ + +#include "lauxlib.h" +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/msg.h" +#include "upb/msgfactory.h" + +/* Lua 5.1/5.2 compatibility code. */ +#if LUA_VERSION_NUM == 501 + +#define lua_rawlen lua_objlen + +/* Lua >= 5.2's getuservalue/setuservalue functions do not exist in prior + * versions but the older function lua_getfenv() can provide 100% of its + * capabilities (the reverse is not true). */ +#define lua_getuservalue(L, index) lua_getfenv(L, index) +#define lua_setuservalue(L, index) lua_setfenv(L, index) + +void *luaL_testudata(lua_State *L, int ud, const char *tname); + +#define lupb_setfuncs(L, l) luaL_register(L, NULL, l) + +#elif LUA_VERSION_NUM == 502 + +int luaL_typerror(lua_State *L, int narg, const char *tname); + +#define lupb_setfuncs(L, l) luaL_setfuncs(L, l, 0) + +#else +#error Only Lua 5.1 and 5.2 are supported +#endif + +#define lupb_assert(L, predicate) \ + if (!(predicate)) \ + luaL_error(L, "internal error: %s, %s:%d ", #predicate, __FILE__, __LINE__); + +/* Function for initializing the core library. This function is idempotent, + * and should be called at least once before calling any of the functions that + * construct core upb types. */ +int luaopen_upb(lua_State *L); + +/* Gets or creates a package table for a C module that is uniquely identified by + * "ptr". The easiest way to supply a unique "ptr" is to pass the address of a + * static variable private in the module's .c file. + * + * If this module has already been registered in this lua_State, pushes it and + * returns true. + * + * Otherwise, creates a new module table for this module with the given name, + * pushes it, and registers the given top-level functions in it. It also sets + * it as a global variable, but only if the current version of Lua expects that + * (ie Lua 5.1/LuaJIT). + * + * If "false" is returned, the caller is guaranteed that this lib has not been + * registered in this Lua state before (regardless of any funny business the + * user might have done to the global state), so the caller can safely perform + * one-time initialization. */ +bool lupb_openlib(lua_State *L, void *ptr, const char *name, + const luaL_Reg *funcs); + +/* Custom check/push functions. Unlike the Lua equivalents, they are pinned to + * specific types (instead of lua_Number, etc), and do not allow any implicit + * conversion or data loss. */ +int64_t lupb_checkint64(lua_State *L, int narg); +int32_t lupb_checkint32(lua_State *L, int narg); +uint64_t lupb_checkuint64(lua_State *L, int narg); +uint32_t lupb_checkuint32(lua_State *L, int narg); +double lupb_checkdouble(lua_State *L, int narg); +float lupb_checkfloat(lua_State *L, int narg); +bool lupb_checkbool(lua_State *L, int narg); +const char *lupb_checkstring(lua_State *L, int narg, size_t *len); +const char *lupb_checkname(lua_State *L, int narg); + +void lupb_pushint64(lua_State *L, int64_t val); +void lupb_pushint32(lua_State *L, int32_t val); +void lupb_pushuint64(lua_State *L, uint64_t val); +void lupb_pushuint32(lua_State *L, uint32_t val); +void lupb_pushdouble(lua_State *L, double val); +void lupb_pushfloat(lua_State *L, float val); + +/* Registers a type with the given name, methods, and metamethods. */ +void lupb_register_type(lua_State *L, const char *name, const luaL_Reg *m, + const luaL_Reg *mm); + +/* Checks the given upb_status and throws a Lua error if it is not ok. */ +void lupb_checkstatus(lua_State *L, upb_status *s); + + +/** From def.c. ***************************************************************/ + +upb_fieldtype_t lupb_checkfieldtype(lua_State *L, int narg); + +const upb_msgdef *lupb_msgdef_check(lua_State *L, int narg); +const upb_enumdef *lupb_enumdef_check(lua_State *L, int narg); +const upb_fielddef *lupb_fielddef_check(lua_State *L, int narg); +upb_symtab *lupb_symtab_check(lua_State *L, int narg); + +void lupb_def_registertypes(lua_State *L); + + +/** From msg.c. ***************************************************************/ + +struct lupb_msgclass; +typedef struct lupb_msgclass lupb_msgclass; + +upb_arena *lupb_arena_check(lua_State *L, int narg); +int lupb_arena_new(lua_State *L); +upb_arena *lupb_arena_get(lua_State *L); +int lupb_msg_pushref(lua_State *L, int msgclass, void *msg); +const upb_msg *lupb_msg_checkmsg(lua_State *L, int narg, + const lupb_msgclass *lmsgclass); +upb_msg *lupb_msg_checkmsg2(lua_State *L, int narg, + const upb_msglayout **layout); + +const lupb_msgclass *lupb_msgclass_check(lua_State *L, int narg); +const upb_msglayout *lupb_msgclass_getlayout(lua_State *L, int narg); +const upb_msgdef *lupb_msgclass_getmsgdef(const lupb_msgclass *lmsgclass); +upb_msgfactory *lupb_msgclass_getfactory(const lupb_msgclass *lmsgclass); +void lupb_msg_registertypes(lua_State *L); + +#endif /* UPB_LUA_UPB_H_ */ diff --git a/third_party/upb/upb/bindings/lua/upb.lua b/third_party/upb/upb/bindings/lua/upb.lua new file mode 100644 index 00000000000..728852e997a --- /dev/null +++ b/third_party/upb/upb/bindings/lua/upb.lua @@ -0,0 +1,172 @@ + +-- Before calling require on "upb_c", we need to load the same library +-- as RTLD_GLOBAL, for the benefit of other C extensions that depend on +-- C functions in the core. +-- +-- This has to happen *before* the require call, because if the module +-- is loaded RTLD_LOCAL first, a subsequent load as RTLD_GLOBAL won't +-- have the proper effect, at least on some platforms. +local so = package.searchpath and package.searchpath("upb_c", package.cpath) +if so then + package.loadlib(so, "*") +end + +local upb = require("upb_c") + +-- A convenience function for building/linking/freezing defs +-- while maintaining their original order. +-- +-- Sample usage: +-- local m1, m2 = upb.build_defs{ +-- upb.MessageDef{full_name = "M1", fields = { +-- upb.FieldDef{ +-- name = "m2", +-- number = 1, +-- type = upb.TYPE_MESSAGE, +-- subdef_name = ".M2" +-- }, +-- } +-- }, +-- upb.MessageDef{full_name = "M2"} +-- } +upb.build_defs = function(defs) + upb.SymbolTable(defs) + -- Lua 5.2 puts unpack in the table library. + return (unpack or table.unpack)(defs) +end + +local ipairs_iter = function(array, last_index) + local next_index = last_index + 1 + if next_index > #array then + return nil + end + return next_index, array[next_index] +end + +-- For iterating over the indexes and values of a upb.Array. +-- +-- for i, val in upb.ipairs(array) do +-- -- ... +-- end +upb.ipairs = function(array) + return ipairs_iter, array, 0 +end + +local set_named = function(obj, init) + for k, v in pairs(init) do + local func = obj["set_" .. k] + if not func then + error("Cannot set member: " .. k) + end + func(obj, v) + end +end + +-- Capture references to the functions we're wrapping. +local RealFieldDef = upb.FieldDef +local RealEnumDef = upb.EnumDef +local RealMessageDef = upb.MessageDef +local RealOneofDef = upb.OneofDef +local RealSymbolTable = upb.SymbolTable + +-- FieldDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5, +-- type=upb.TYPE_INT32, default_value=12, type_name="Foo"} +upb.FieldDef = function(init) + local f = RealFieldDef() + + if init then + -- Other members are often dependent on type, so set that first. + if init.type then + f:set_type(init.type) + init.type = nil + end + + set_named(f, init) + end + + return f +end + + +-- MessageDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}} +upb.MessageDef = function(init) + local m = RealMessageDef() + + if init then + for _, f in pairs(init.fields or {}) do + m:add(f) + end + init.fields = nil + + set_named(m, init) + end + + return m +end + +-- EnumDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.EnumDef{full_name="MyEnum", +-- values={ +-- {"FOO_VALUE_1", 1}, +-- {"FOO_VALUE_2", 2} +-- } +-- } +upb.EnumDef = function(init) + local e = RealEnumDef() + + if init then + for _, val in pairs(init.values or {}) do + e:add(val[1], val[2]) + end + init.values = nil + + set_named(e, init) + end + + return e +end + +-- OneofDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.OneofDef{name="foo", fields={...}} +upb.OneofDef = function(init) + local o = RealOneofDef() + + if init then + for _, val in pairs(init.fields or {}) do + o:add(val) + end + init.fields = nil + + set_named(o, init) + end + + return o +end + +-- SymbolTable constructor; a wrapper around the real constructor that can +-- add an initial set of defs. +upb.SymbolTable = function(defs) + local s = RealSymbolTable() + + if defs then + s:add(defs) + end + + return s +end + +return upb diff --git a/third_party/upb/upb/bindings/lua/upb/pb.c b/third_party/upb/upb/bindings/lua/upb/pb.c new file mode 100644 index 00000000000..266bd974c9e --- /dev/null +++ b/third_party/upb/upb/bindings/lua/upb/pb.c @@ -0,0 +1,56 @@ +/* +** require("upb.pb") -- A Lua extension for upb.pb. +** +** Exposes all the types defined in upb/pb/{*}.h +** Also defines a few convenience functions on top. +*/ + +#include "upb/bindings/lua/upb.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#define LUPB_PBDECODERMETHOD "lupb.pb.decodermethod" + +static int lupb_pb_decode(lua_State *L) { + size_t len; + const upb_msglayout *layout; + upb_msg *msg = lupb_msg_checkmsg2(L, 1, &layout); + const char *pb = lua_tolstring(L, 2, &len); + + upb_decode(pb, len, msg, layout, lupb_arena_get(L)); + /* TODO(haberman): check for error. */ + + return 0; +} + +static int lupb_pb_encode(lua_State *L) { + const upb_msglayout *layout; + const upb_msg *msg = lupb_msg_checkmsg2(L, 1, &layout); + upb_arena *arena = upb_arena_new(); + size_t size; + char *result; + + result = upb_encode(msg, (const void*)layout, arena, &size); + + /* Free resources before we potentially bail on error. */ + lua_pushlstring(L, result, size); + upb_arena_free(arena); + /* TODO(haberman): check for error. */ + + return 1; +} + +static const struct luaL_Reg toplevel_m[] = { + {"decode", lupb_pb_decode}, + {"encode", lupb_pb_encode}, + {NULL, NULL} +}; + +int luaopen_upb_pb_c(lua_State *L) { + static char module_key; + if (lupb_openlib(L, &module_key, "upb.pb_c", toplevel_m)) { + return 1; + } + + return 1; +} diff --git a/third_party/upb/upb/bindings/lua/upb/pb.lua b/third_party/upb/upb/bindings/lua/upb/pb.lua new file mode 100644 index 00000000000..b865902d3a5 --- /dev/null +++ b/third_party/upb/upb/bindings/lua/upb/pb.lua @@ -0,0 +1,3 @@ + +require "upb" +return require "upb.pb_c" diff --git a/third_party/upb/upb/bindings/stdc++/string.h b/third_party/upb/upb/bindings/stdc++/string.h new file mode 100644 index 00000000000..c3465481a34 --- /dev/null +++ b/third_party/upb/upb/bindings/stdc++/string.h @@ -0,0 +1,69 @@ + +#ifndef UPB_STDCPP_H_ +#define UPB_STDCPP_H_ + +#include "upb/sink.h" + +#include "upb/port_def.inc" + +namespace upb { + +template +class FillStringHandler { + public: + static void SetHandler(upb_byteshandler* handler) { + upb_byteshandler_setstartstr(handler, &FillStringHandler::StartString, + NULL); + upb_byteshandler_setstring(handler, &FillStringHandler::StringBuf, NULL); + } + + private: + // TODO(haberman): add UpbBind/UpbMakeHandler support to BytesHandler so these + // can be prettier callbacks. + static void* StartString(void *c, const void *hd, size_t size) { + UPB_UNUSED(hd); + UPB_UNUSED(size); + + T* str = static_cast(c); + str->clear(); + return c; + } + + static size_t StringBuf(void* c, const void* hd, const char* buf, size_t n, + const upb_bufhandle* h) { + UPB_UNUSED(hd); + UPB_UNUSED(h); + + T* str = static_cast(c); + try { + str->append(buf, n); + return n; + } catch (const std::exception&) { + return 0; + } + } +}; + +class StringSink { + public: + template + explicit StringSink(T* target) { + // TODO(haberman): we need to avoid rebuilding a new handler every time, + // but with class globals disallowed for google3 C++ this is tricky. + upb_byteshandler_init(&handler_); + FillStringHandler::SetHandler(&handler_); + input_.Reset(&handler_, target); + } + + BytesSink input() { return input_; } + + private: + upb_byteshandler handler_; + BytesSink input_; +}; + +} // namespace upb + +#include "upb/port_undef.inc" + +#endif // UPB_STDCPP_H_ diff --git a/third_party/upb/upb/decode.c b/third_party/upb/upb/decode.c new file mode 100644 index 00000000000..88d4bb4fb6d --- /dev/null +++ b/third_party/upb/upb/decode.c @@ -0,0 +1,604 @@ + +#include +#include "upb/upb.h" +#include "upb/decode.h" + +#include "upb/port_def.inc" + +/* Maps descriptor type -> upb field type. */ +const uint8_t upb_desctype_to_fieldtype[] = { + UPB_WIRE_TYPE_END_GROUP, /* ENDGROUP */ + UPB_TYPE_DOUBLE, /* DOUBLE */ + UPB_TYPE_FLOAT, /* FLOAT */ + UPB_TYPE_INT64, /* INT64 */ + UPB_TYPE_UINT64, /* UINT64 */ + UPB_TYPE_INT32, /* INT32 */ + UPB_TYPE_UINT64, /* FIXED64 */ + UPB_TYPE_UINT32, /* FIXED32 */ + UPB_TYPE_BOOL, /* BOOL */ + UPB_TYPE_STRING, /* STRING */ + UPB_TYPE_MESSAGE, /* GROUP */ + UPB_TYPE_MESSAGE, /* MESSAGE */ + UPB_TYPE_BYTES, /* BYTES */ + UPB_TYPE_UINT32, /* UINT32 */ + UPB_TYPE_ENUM, /* ENUM */ + UPB_TYPE_INT32, /* SFIXED32 */ + UPB_TYPE_INT64, /* SFIXED64 */ + UPB_TYPE_INT32, /* SINT32 */ + UPB_TYPE_INT64, /* SINT64 */ +}; + +/* Data pertaining to the parse. */ +typedef struct { + const char *ptr; /* Current parsing position. */ + const char *field_start; /* Start of this field. */ + const char *limit; /* End of delimited region or end of buffer. */ + upb_arena *arena; + int depth; + uint32_t end_group; /* Set to field number of END_GROUP tag, if any. */ +} upb_decstate; + +/* Data passed by value to each parsing function. */ +typedef struct { + char *msg; + const upb_msglayout *layout; + upb_decstate *state; +} upb_decframe; + +#define CHK(x) if (!(x)) { return 0; } + +static bool upb_skip_unknowngroup(upb_decstate *d, int field_number); +static bool upb_decode_message(upb_decstate *d, char *msg, + const upb_msglayout *l); + +static bool upb_decode_varint(const char **ptr, const char *limit, + uint64_t *val) { + uint8_t byte; + int bitpos = 0; + const char *p = *ptr; + *val = 0; + + do { + CHK(bitpos < 70 && p < limit); + byte = *p; + *val |= (uint64_t)(byte & 0x7F) << bitpos; + p++; + bitpos += 7; + } while (byte & 0x80); + + *ptr = p; + return true; +} + +static bool upb_decode_varint32(const char **ptr, const char *limit, + uint32_t *val) { + uint64_t u64; + CHK(upb_decode_varint(ptr, limit, &u64) && u64 <= UINT32_MAX); + *val = (uint32_t)u64; + return true; +} + +static bool upb_decode_64bit(const char **ptr, const char *limit, + uint64_t *val) { + CHK(limit - *ptr >= 8); + memcpy(val, *ptr, 8); + *ptr += 8; + return true; +} + +static bool upb_decode_32bit(const char **ptr, const char *limit, + uint32_t *val) { + CHK(limit - *ptr >= 4); + memcpy(val, *ptr, 4); + *ptr += 4; + return true; +} + +static int32_t upb_zzdecode_32(uint32_t n) { + return (n >> 1) ^ -(int32_t)(n & 1); +} + +static int64_t upb_zzdecode_64(uint64_t n) { + return (n >> 1) ^ -(int64_t)(n & 1); +} + +static bool upb_decode_string(const char **ptr, const char *limit, + int *outlen) { + uint32_t len; + + CHK(upb_decode_varint32(ptr, limit, &len) && + len < INT32_MAX && + limit - *ptr >= (int32_t)len); + + *outlen = len; + return true; +} + +static void upb_set32(void *msg, size_t ofs, uint32_t val) { + memcpy((char*)msg + ofs, &val, sizeof(val)); +} + +static bool upb_append_unknown(upb_decstate *d, upb_decframe *frame) { + upb_msg_addunknown(frame->msg, d->field_start, d->ptr - d->field_start, + d->arena); + return true; +} + + +static bool upb_skip_unknownfielddata(upb_decstate *d, uint32_t tag, + uint32_t group_fieldnum) { + switch (tag & 7) { + case UPB_WIRE_TYPE_VARINT: { + uint64_t val; + return upb_decode_varint(&d->ptr, d->limit, &val); + } + case UPB_WIRE_TYPE_32BIT: { + uint32_t val; + return upb_decode_32bit(&d->ptr, d->limit, &val); + } + case UPB_WIRE_TYPE_64BIT: { + uint64_t val; + return upb_decode_64bit(&d->ptr, d->limit, &val); + } + case UPB_WIRE_TYPE_DELIMITED: { + int len; + CHK(upb_decode_string(&d->ptr, d->limit, &len)); + d->ptr += len; + return true; + } + case UPB_WIRE_TYPE_START_GROUP: + return upb_skip_unknowngroup(d, tag >> 3); + case UPB_WIRE_TYPE_END_GROUP: + return (tag >> 3) == group_fieldnum; + } + return false; +} + +static bool upb_skip_unknowngroup(upb_decstate *d, int field_number) { + while (d->ptr < d->limit && d->end_group == 0) { + uint32_t tag = 0; + CHK(upb_decode_varint32(&d->ptr, d->limit, &tag)); + CHK(upb_skip_unknownfielddata(d, tag, field_number)); + } + + CHK(d->end_group == field_number); + d->end_group = 0; + return true; +} + +static bool upb_array_grow(upb_array *arr, size_t elements, size_t elem_size, + upb_arena *arena) { + size_t needed = arr->len + elements; + size_t new_size = UPB_MAX(arr->size, 8); + size_t new_bytes; + size_t old_bytes; + void *new_data; + upb_alloc *alloc = upb_arena_alloc(arena); + + while (new_size < needed) { + new_size *= 2; + } + + old_bytes = arr->len * elem_size; + new_bytes = new_size * elem_size; + new_data = upb_realloc(alloc, arr->data, old_bytes, new_bytes); + CHK(new_data); + + arr->data = new_data; + arr->size = new_size; + return true; +} + +static void *upb_array_reserve(upb_array *arr, size_t elements, + size_t elem_size, upb_arena *arena) { + if (arr->size - arr->len < elements) { + CHK(upb_array_grow(arr, elements, elem_size, arena)); + } + return (char*)arr->data + (arr->len * elem_size); +} + +bool upb_array_add(upb_array *arr, size_t elements, size_t elem_size, + const void *data, upb_arena *arena) { + void *dest = upb_array_reserve(arr, elements, elem_size, arena); + + CHK(dest); + arr->len += elements; + memcpy(dest, data, elements * elem_size); + + return true; +} + +static upb_array *upb_getarr(upb_decframe *frame, + const upb_msglayout_field *field) { + UPB_ASSERT(field->label == UPB_LABEL_REPEATED); + return *(upb_array**)&frame->msg[field->offset]; +} + +static upb_array *upb_getorcreatearr(upb_decframe *frame, + const upb_msglayout_field *field) { + upb_array *arr = upb_getarr(frame, field); + + if (!arr) { + arr = upb_array_new(frame->state->arena); + CHK(arr); + *(upb_array**)&frame->msg[field->offset] = arr; + } + + return arr; +} + +static upb_msg *upb_getorcreatemsg(upb_decframe *frame, + const upb_msglayout_field *field, + const upb_msglayout **subm) { + upb_msg **submsg = (void*)(frame->msg + field->offset); + *subm = frame->layout->submsgs[field->submsg_index]; + + UPB_ASSERT(field->label != UPB_LABEL_REPEATED); + + if (!*submsg) { + *submsg = upb_msg_new(*subm, frame->state->arena); + CHK(*submsg); + } + + return *submsg; +} + +static upb_msg *upb_addmsg(upb_decframe *frame, + const upb_msglayout_field *field, + const upb_msglayout **subm) { + upb_msg *submsg; + upb_array *arr = upb_getorcreatearr(frame, field); + + *subm = frame->layout->submsgs[field->submsg_index]; + submsg = upb_msg_new(*subm, frame->state->arena); + CHK(submsg); + upb_array_add(arr, 1, sizeof(submsg), &submsg, frame->state->arena); + + return submsg; +} + +static void upb_sethasbit(upb_decframe *frame, + const upb_msglayout_field *field) { + int32_t hasbit = field->presence; + UPB_ASSERT(field->presence > 0); + frame->msg[hasbit / 8] |= (1 << (hasbit % 8)); +} + +static void upb_setoneofcase(upb_decframe *frame, + const upb_msglayout_field *field) { + UPB_ASSERT(field->presence < 0); + upb_set32(frame->msg, ~field->presence, field->number); +} + +static bool upb_decode_addval(upb_decframe *frame, + const upb_msglayout_field *field, void *val, + size_t size) { + char *field_mem = frame->msg + field->offset; + upb_array *arr; + + if (field->label == UPB_LABEL_REPEATED) { + arr = upb_getorcreatearr(frame, field); + CHK(arr); + field_mem = upb_array_reserve(arr, 1, size, frame->state->arena); + CHK(field_mem); + } + + memcpy(field_mem, val, size); + return true; +} + +static void upb_decode_setpresent(upb_decframe *frame, + const upb_msglayout_field *field) { + if (field->label == UPB_LABEL_REPEATED) { + upb_array *arr = upb_getarr(frame, field); + UPB_ASSERT(arr->len < arr->size); + arr->len++; + } else if (field->presence < 0) { + upb_setoneofcase(frame, field); + } else if (field->presence > 0) { + upb_sethasbit(frame, field); + } +} + +static bool upb_decode_msgfield(upb_decstate *d, upb_msg *msg, + const upb_msglayout *layout, int limit) { + const char* saved_limit = d->limit; + d->limit = d->ptr + limit; + CHK(--d->depth >= 0); + upb_decode_message(d, msg, layout); + d->depth++; + d->limit = saved_limit; + CHK(d->end_group == 0); + return true; +} + +static bool upb_decode_groupfield(upb_decstate *d, upb_msg *msg, + const upb_msglayout *layout, + int field_number) { + CHK(--d->depth >= 0); + upb_decode_message(d, msg, layout); + d->depth++; + CHK(d->end_group == field_number); + d->end_group = 0; + return true; +} + +static bool upb_decode_varintfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + uint64_t val; + CHK(upb_decode_varint(&d->ptr, d->limit, &val)); + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + CHK(upb_decode_addval(frame, field, &val, sizeof(val))); + break; + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_UINT32: + case UPB_DESCRIPTOR_TYPE_ENUM: { + uint32_t val32 = (uint32_t)val; + CHK(upb_decode_addval(frame, field, &val32, sizeof(val32))); + break; + } + case UPB_DESCRIPTOR_TYPE_BOOL: { + bool valbool = val != 0; + CHK(upb_decode_addval(frame, field, &valbool, sizeof(valbool))); + break; + } + case UPB_DESCRIPTOR_TYPE_SINT32: { + int32_t decoded = upb_zzdecode_32((uint32_t)val); + CHK(upb_decode_addval(frame, field, &decoded, sizeof(decoded))); + break; + } + case UPB_DESCRIPTOR_TYPE_SINT64: { + int64_t decoded = upb_zzdecode_64(val); + CHK(upb_decode_addval(frame, field, &decoded, sizeof(decoded))); + break; + } + default: + return upb_append_unknown(d, frame); + } + + upb_decode_setpresent(frame, field); + return true; +} + +static bool upb_decode_64bitfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + uint64_t val; + CHK(upb_decode_64bit(&d->ptr, d->limit, &val)); + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + case UPB_DESCRIPTOR_TYPE_FIXED64: + case UPB_DESCRIPTOR_TYPE_SFIXED64: + CHK(upb_decode_addval(frame, field, &val, sizeof(val))); + break; + default: + return upb_append_unknown(d, frame); + } + + upb_decode_setpresent(frame, field); + return true; +} + +static bool upb_decode_32bitfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + uint32_t val; + CHK(upb_decode_32bit(&d->ptr, d->limit, &val)); + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_FLOAT: + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + CHK(upb_decode_addval(frame, field, &val, sizeof(val))); + break; + default: + return upb_append_unknown(d, frame); + } + + upb_decode_setpresent(frame, field); + return true; +} + +static bool upb_decode_fixedpacked(upb_decstate *d, upb_array *arr, + uint32_t len, int elem_size) { + size_t elements = len / elem_size; + + CHK((size_t)(elements * elem_size) == len); + CHK(upb_array_add(arr, elements, elem_size, d->ptr, d->arena)); + d->ptr += len; + + return true; +} + +static upb_strview upb_decode_strfield(upb_decstate *d, uint32_t len) { + upb_strview ret; + ret.data = d->ptr; + ret.size = len; + d->ptr += len; + return ret; +} + +static bool upb_decode_toarray(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field, int len) { + upb_array *arr = upb_getorcreatearr(frame, field); + CHK(arr); + +#define VARINT_CASE(ctype, decode) \ + VARINT_CASE_EX(ctype, decode, decode) + +#define VARINT_CASE_EX(ctype, decode, dtype) \ + { \ + const char *ptr = d->ptr; \ + const char *limit = ptr + len; \ + while (ptr < limit) { \ + uint64_t val; \ + ctype decoded; \ + CHK(upb_decode_varint(&ptr, limit, &val)); \ + decoded = (decode)((dtype)val); \ + CHK(upb_array_add(arr, 1, sizeof(decoded), &decoded, d->arena)); \ + } \ + d->ptr = ptr; \ + return true; \ + } + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview str = upb_decode_strfield(d, len); + return upb_array_add(arr, 1, sizeof(str), &str, d->arena); + } + case UPB_DESCRIPTOR_TYPE_FLOAT: + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + return upb_decode_fixedpacked(d, arr, len, sizeof(int32_t)); + case UPB_DESCRIPTOR_TYPE_DOUBLE: + case UPB_DESCRIPTOR_TYPE_FIXED64: + case UPB_DESCRIPTOR_TYPE_SFIXED64: + return upb_decode_fixedpacked(d, arr, len, sizeof(int64_t)); + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_UINT32: + case UPB_DESCRIPTOR_TYPE_ENUM: + VARINT_CASE(uint32_t, uint32_t); + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + VARINT_CASE(uint64_t, uint64_t); + case UPB_DESCRIPTOR_TYPE_BOOL: + VARINT_CASE(bool, bool); + case UPB_DESCRIPTOR_TYPE_SINT32: + VARINT_CASE_EX(int32_t, upb_zzdecode_32, uint32_t); + case UPB_DESCRIPTOR_TYPE_SINT64: + VARINT_CASE_EX(int64_t, upb_zzdecode_64, uint64_t); + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + const upb_msglayout *subm; + upb_msg *submsg = upb_addmsg(frame, field, &subm); + CHK(submsg); + return upb_decode_msgfield(d, submsg, subm, len); + } + case UPB_DESCRIPTOR_TYPE_GROUP: + return upb_append_unknown(d, frame); + } +#undef VARINT_CASE + UPB_UNREACHABLE(); +} + +static bool upb_decode_delimitedfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + int len; + + CHK(upb_decode_string(&d->ptr, d->limit, &len)); + + if (field->label == UPB_LABEL_REPEATED) { + return upb_decode_toarray(d, frame, field, len); + } else { + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview str = upb_decode_strfield(d, len); + CHK(upb_decode_addval(frame, field, &str, sizeof(str))); + break; + } + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + const upb_msglayout *subm; + upb_msg *submsg = upb_getorcreatemsg(frame, field, &subm); + CHK(submsg); + CHK(upb_decode_msgfield(d, submsg, subm, len)); + break; + } + default: + /* TODO(haberman): should we accept the last element of a packed? */ + d->ptr += len; + return upb_append_unknown(d, frame); + } + upb_decode_setpresent(frame, field); + return true; + } +} + +static const upb_msglayout_field *upb_find_field(const upb_msglayout *l, + uint32_t field_number) { + /* Lots of optimization opportunities here. */ + int i; + for (i = 0; i < l->field_count; i++) { + if (l->fields[i].number == field_number) { + return &l->fields[i]; + } + } + + return NULL; /* Unknown field. */ +} + +static bool upb_decode_field(upb_decstate *d, upb_decframe *frame) { + uint32_t tag; + const upb_msglayout_field *field; + int field_number; + + d->field_start = d->ptr; + CHK(upb_decode_varint32(&d->ptr, d->limit, &tag)); + field_number = tag >> 3; + field = upb_find_field(frame->layout, field_number); + + if (field) { + switch (tag & 7) { + case UPB_WIRE_TYPE_VARINT: + return upb_decode_varintfield(d, frame, field); + case UPB_WIRE_TYPE_32BIT: + return upb_decode_32bitfield(d, frame, field); + case UPB_WIRE_TYPE_64BIT: + return upb_decode_64bitfield(d, frame, field); + case UPB_WIRE_TYPE_DELIMITED: + return upb_decode_delimitedfield(d, frame, field); + case UPB_WIRE_TYPE_START_GROUP: { + const upb_msglayout *layout; + upb_msg *group; + + if (field->label == UPB_LABEL_REPEATED) { + group = upb_addmsg(frame, field, &layout); + } else { + group = upb_getorcreatemsg(frame, field, &layout); + } + + return upb_decode_groupfield(d, group, layout, field_number); + } + case UPB_WIRE_TYPE_END_GROUP: + d->end_group = field_number; + return true; + default: + CHK(false); + } + } else { + CHK(field_number != 0); + CHK(upb_skip_unknownfielddata(d, tag, -1)); + CHK(upb_append_unknown(d, frame)); + return true; + } +} + +static bool upb_decode_message(upb_decstate *d, char *msg, const upb_msglayout *l) { + upb_decframe frame; + frame.msg = msg; + frame.layout = l; + frame.state = d; + + while (d->ptr < d->limit) { + CHK(upb_decode_field(d, &frame)); + } + + return true; +} + +bool upb_decode(const char *buf, size_t size, void *msg, const upb_msglayout *l, + upb_arena *arena) { + upb_decstate state; + state.ptr = buf; + state.limit = buf + size; + state.arena = arena; + state.depth = 64; + state.end_group = 0; + + CHK(upb_decode_message(&state, msg, l)); + return state.end_group == 0; +} + +#undef CHK diff --git a/third_party/upb/upb/decode.h b/third_party/upb/upb/decode.h new file mode 100644 index 00000000000..9de8638de5c --- /dev/null +++ b/third_party/upb/upb/decode.h @@ -0,0 +1,21 @@ +/* +** upb_decode: parsing into a upb_msg using a upb_msglayout. +*/ + +#ifndef UPB_DECODE_H_ +#define UPB_DECODE_H_ + +#include "upb/msg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool upb_decode(const char *buf, size_t size, upb_msg *msg, + const upb_msglayout *l, upb_arena *arena); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_DECODE_H_ */ diff --git a/third_party/upb/upb/def.c b/third_party/upb/upb/def.c new file mode 100644 index 00000000000..98aead6b674 --- /dev/null +++ b/third_party/upb/upb/def.c @@ -0,0 +1,1756 @@ + +#include "upb/def.h" + +#include +#include +#include +#include +#include "google/protobuf/descriptor.upb.h" + +#include "upb/port_def.inc" + +typedef struct { + size_t len; + char str[1]; /* Null-terminated string data follows. */ +} str_t; + +static str_t *newstr(upb_alloc *alloc, const char *data, size_t len) { + str_t *ret = upb_malloc(alloc, sizeof(*ret) + len); + if (!ret) return NULL; + ret->len = len; + memcpy(ret->str, data, len); + ret->str[len] = '\0'; + return ret; +} + +struct upb_fielddef { + const upb_filedef *file; + const upb_msgdef *msgdef; + const char *full_name; + union { + int64_t sint; + uint64_t uint; + double dbl; + float flt; + bool boolean; + str_t *str; + } defaultval; + const upb_oneofdef *oneof; + union { + const upb_msgdef *msgdef; + const upb_enumdef *enumdef; + const google_protobuf_FieldDescriptorProto *unresolved; + } sub; + uint32_t number_; + uint32_t index_; + uint32_t selector_base; /* Used to index into a upb::Handlers table. */ + bool is_extension_; + bool lazy_; + bool packed_; + upb_descriptortype_t type_; + upb_label_t label_; +}; + +struct upb_msgdef { + const upb_filedef *file; + const char *full_name; + uint32_t selector_count; + uint32_t submsg_field_count; + + /* Tables for looking up fields by number and name. */ + upb_inttable itof; + upb_strtable ntof; + + const upb_fielddef *fields; + const upb_oneofdef *oneofs; + int field_count; + int oneof_count; + + /* Is this a map-entry message? */ + bool map_entry; + upb_wellknowntype_t well_known_type; + + /* TODO(haberman): proper extension ranges (there can be multiple). */ +}; + +struct upb_enumdef { + const upb_filedef *file; + const char *full_name; + upb_strtable ntoi; + upb_inttable iton; + int32_t defaultval; +}; + +struct upb_oneofdef { + const upb_msgdef *parent; + const char *full_name; + uint32_t index; + upb_strtable ntof; + upb_inttable itof; +}; + +struct upb_filedef { + const char *name; + const char *package; + const char *phpprefix; + const char *phpnamespace; + upb_syntax_t syntax; + + const upb_filedef **deps; + const upb_msgdef *msgs; + const upb_enumdef *enums; + const upb_fielddef *exts; + + int dep_count; + int msg_count; + int enum_count; + int ext_count; +}; + +struct upb_symtab { + upb_arena *arena; + upb_strtable syms; /* full_name -> packed def ptr */ + upb_strtable files; /* file_name -> upb_filedef* */ +}; + +/* Inside a symtab we store tagged pointers to specific def types. */ +typedef enum { + UPB_DEFTYPE_MSG = 0, + UPB_DEFTYPE_ENUM = 1, + UPB_DEFTYPE_FIELD = 2, + UPB_DEFTYPE_ONEOF = 3 +} upb_deftype_t; + +static const void *unpack_def(upb_value v, upb_deftype_t type) { + uintptr_t num = (uintptr_t)upb_value_getconstptr(v); + return (num & 3) == type ? (const void*)(num & ~3) : NULL; +} + +static upb_value pack_def(const void *ptr, upb_deftype_t type) { + uintptr_t num = (uintptr_t)ptr | type; + return upb_value_constptr((const void*)num); +} + +/* isalpha() etc. from are locale-dependent, which we don't want. */ +static bool upb_isbetween(char c, char low, char high) { + return c >= low && c <= high; +} + +static bool upb_isletter(char c) { + return upb_isbetween(c, 'A', 'Z') || upb_isbetween(c, 'a', 'z') || c == '_'; +} + +static bool upb_isalphanum(char c) { + return upb_isletter(c) || upb_isbetween(c, '0', '9'); +} + +static bool upb_isident(upb_strview name, bool full, upb_status *s) { + const char *str = name.data; + size_t len = name.size; + bool start = true; + size_t i; + for (i = 0; i < len; i++) { + char c = str[i]; + if (c == '.') { + if (start || !full) { + upb_status_seterrf(s, "invalid name: unexpected '.' (%s)", str); + return false; + } + start = true; + } else if (start) { + if (!upb_isletter(c)) { + upb_status_seterrf( + s, "invalid name: path components must start with a letter (%s)", + str); + return false; + } + start = false; + } else { + if (!upb_isalphanum(c)) { + upb_status_seterrf(s, "invalid name: non-alphanumeric character (%s)", + str); + return false; + } + } + } + return !start; +} + +static const char *shortdefname(const char *fullname) { + const char *p; + + if (fullname == NULL) { + return NULL; + } else if ((p = strrchr(fullname, '.')) == NULL) { + /* No '.' in the name, return the full string. */ + return fullname; + } else { + /* Return one past the last '.'. */ + return p + 1; + } +} + +/* All submessage fields are lower than all other fields. + * Secondly, fields are increasing in order. */ +uint32_t field_rank(const upb_fielddef *f) { + uint32_t ret = upb_fielddef_number(f); + const uint32_t high_bit = 1 << 30; + UPB_ASSERT(ret < high_bit); + if (!upb_fielddef_issubmsg(f)) + ret |= high_bit; + return ret; +} + +int cmp_fields(const void *p1, const void *p2) { + const upb_fielddef *f1 = *(upb_fielddef*const*)p1; + const upb_fielddef *f2 = *(upb_fielddef*const*)p2; + return field_rank(f1) - field_rank(f2); +} + +/* A few implementation details of handlers. We put these here to avoid + * a def -> handlers dependency. */ + +#define UPB_STATIC_SELECTOR_COUNT 3 /* Warning: also in upb/handlers.h. */ + +static uint32_t upb_handlers_selectorbaseoffset(const upb_fielddef *f) { + return upb_fielddef_isseq(f) ? 2 : 0; +} + +static uint32_t upb_handlers_selectorcount(const upb_fielddef *f) { + uint32_t ret = 1; + if (upb_fielddef_isseq(f)) ret += 2; /* STARTSEQ/ENDSEQ */ + if (upb_fielddef_isstring(f)) ret += 2; /* [STRING]/STARTSTR/ENDSTR */ + if (upb_fielddef_issubmsg(f)) { + /* ENDSUBMSG (STARTSUBMSG is at table beginning) */ + ret += 0; + if (upb_fielddef_lazy(f)) { + /* STARTSTR/ENDSTR/STRING (for lazy) */ + ret += 3; + } + } + return ret; +} + +static bool assign_msg_indices(upb_msgdef *m, upb_status *s) { + /* Sort fields. upb internally relies on UPB_TYPE_MESSAGE fields having the + * lowest indexes, but we do not publicly guarantee this. */ + upb_msg_field_iter j; + upb_msg_oneof_iter k; + int i; + uint32_t selector; + int n = upb_msgdef_numfields(m); + upb_fielddef **fields; + + if (n == 0) { + m->selector_count = UPB_STATIC_SELECTOR_COUNT; + m->submsg_field_count = 0; + return true; + } + + fields = upb_gmalloc(n * sizeof(*fields)); + if (!fields) { + upb_status_setoom(s); + return false; + } + + m->submsg_field_count = 0; + for(i = 0, upb_msg_field_begin(&j, m); + !upb_msg_field_done(&j); + upb_msg_field_next(&j), i++) { + upb_fielddef *f = upb_msg_iter_field(&j); + UPB_ASSERT(f->msgdef == m); + if (upb_fielddef_issubmsg(f)) { + m->submsg_field_count++; + } + fields[i] = f; + } + + qsort(fields, n, sizeof(*fields), cmp_fields); + + selector = UPB_STATIC_SELECTOR_COUNT + m->submsg_field_count; + for (i = 0; i < n; i++) { + upb_fielddef *f = fields[i]; + f->index_ = i; + f->selector_base = selector + upb_handlers_selectorbaseoffset(f); + selector += upb_handlers_selectorcount(f); + } + m->selector_count = selector; + + for(upb_msg_oneof_begin(&k, m), i = 0; + !upb_msg_oneof_done(&k); + upb_msg_oneof_next(&k), i++) { + upb_oneofdef *o = (upb_oneofdef*)upb_msg_iter_oneof(&k); + o->index = i; + } + + upb_gfree(fields); + return true; +} + +static void assign_msg_wellknowntype(upb_msgdef *m) { + const char *name = upb_msgdef_fullname(m); + if (name == NULL) { + m->well_known_type = UPB_WELLKNOWN_UNSPECIFIED; + return; + } + if (!strcmp(name, "google.protobuf.Any")) { + m->well_known_type = UPB_WELLKNOWN_ANY; + } else if (!strcmp(name, "google.protobuf.FieldMask")) { + m->well_known_type = UPB_WELLKNOWN_FIELDMASK; + } else if (!strcmp(name, "google.protobuf.Duration")) { + m->well_known_type = UPB_WELLKNOWN_DURATION; + } else if (!strcmp(name, "google.protobuf.Timestamp")) { + m->well_known_type = UPB_WELLKNOWN_TIMESTAMP; + } else if (!strcmp(name, "google.protobuf.DoubleValue")) { + m->well_known_type = UPB_WELLKNOWN_DOUBLEVALUE; + } else if (!strcmp(name, "google.protobuf.FloatValue")) { + m->well_known_type = UPB_WELLKNOWN_FLOATVALUE; + } else if (!strcmp(name, "google.protobuf.Int64Value")) { + m->well_known_type = UPB_WELLKNOWN_INT64VALUE; + } else if (!strcmp(name, "google.protobuf.UInt64Value")) { + m->well_known_type = UPB_WELLKNOWN_UINT64VALUE; + } else if (!strcmp(name, "google.protobuf.Int32Value")) { + m->well_known_type = UPB_WELLKNOWN_INT32VALUE; + } else if (!strcmp(name, "google.protobuf.UInt32Value")) { + m->well_known_type = UPB_WELLKNOWN_UINT32VALUE; + } else if (!strcmp(name, "google.protobuf.BoolValue")) { + m->well_known_type = UPB_WELLKNOWN_BOOLVALUE; + } else if (!strcmp(name, "google.protobuf.StringValue")) { + m->well_known_type = UPB_WELLKNOWN_STRINGVALUE; + } else if (!strcmp(name, "google.protobuf.BytesValue")) { + m->well_known_type = UPB_WELLKNOWN_BYTESVALUE; + } else if (!strcmp(name, "google.protobuf.Value")) { + m->well_known_type = UPB_WELLKNOWN_VALUE; + } else if (!strcmp(name, "google.protobuf.ListValue")) { + m->well_known_type = UPB_WELLKNOWN_LISTVALUE; + } else if (!strcmp(name, "google.protobuf.Struct")) { + m->well_known_type = UPB_WELLKNOWN_STRUCT; + } else { + m->well_known_type = UPB_WELLKNOWN_UNSPECIFIED; + } +} + + +/* upb_enumdef ****************************************************************/ + +const char *upb_enumdef_fullname(const upb_enumdef *e) { + return e->full_name; +} + +const char *upb_enumdef_name(const upb_enumdef *e) { + return shortdefname(e->full_name); +} + +const upb_filedef *upb_enumdef_file(const upb_enumdef *e) { + return e->file; +} + +int32_t upb_enumdef_default(const upb_enumdef *e) { + UPB_ASSERT(upb_enumdef_iton(e, e->defaultval)); + return e->defaultval; +} + +int upb_enumdef_numvals(const upb_enumdef *e) { + return upb_strtable_count(&e->ntoi); +} + +void upb_enum_begin(upb_enum_iter *i, const upb_enumdef *e) { + /* We iterate over the ntoi table, to account for duplicate numbers. */ + upb_strtable_begin(i, &e->ntoi); +} + +void upb_enum_next(upb_enum_iter *iter) { upb_strtable_next(iter); } +bool upb_enum_done(upb_enum_iter *iter) { return upb_strtable_done(iter); } + +bool upb_enumdef_ntoi(const upb_enumdef *def, const char *name, + size_t len, int32_t *num) { + upb_value v; + if (!upb_strtable_lookup2(&def->ntoi, name, len, &v)) { + return false; + } + if (num) *num = upb_value_getint32(v); + return true; +} + +const char *upb_enumdef_iton(const upb_enumdef *def, int32_t num) { + upb_value v; + return upb_inttable_lookup32(&def->iton, num, &v) ? + upb_value_getcstr(v) : NULL; +} + +const char *upb_enum_iter_name(upb_enum_iter *iter) { + return upb_strtable_iter_key(iter); +} + +int32_t upb_enum_iter_number(upb_enum_iter *iter) { + return upb_value_getint32(upb_strtable_iter_value(iter)); +} + + +/* upb_fielddef ***************************************************************/ + +const char *upb_fielddef_fullname(const upb_fielddef *f) { + return f->full_name; +} + +upb_fieldtype_t upb_fielddef_type(const upb_fielddef *f) { + switch (f->type_) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + return UPB_TYPE_DOUBLE; + case UPB_DESCRIPTOR_TYPE_FLOAT: + return UPB_TYPE_FLOAT; + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_SINT64: + case UPB_DESCRIPTOR_TYPE_SFIXED64: + return UPB_TYPE_INT64; + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + case UPB_DESCRIPTOR_TYPE_SINT32: + return UPB_TYPE_INT32; + case UPB_DESCRIPTOR_TYPE_UINT64: + case UPB_DESCRIPTOR_TYPE_FIXED64: + return UPB_TYPE_UINT64; + case UPB_DESCRIPTOR_TYPE_UINT32: + case UPB_DESCRIPTOR_TYPE_FIXED32: + return UPB_TYPE_UINT32; + case UPB_DESCRIPTOR_TYPE_ENUM: + return UPB_TYPE_ENUM; + case UPB_DESCRIPTOR_TYPE_BOOL: + return UPB_TYPE_BOOL; + case UPB_DESCRIPTOR_TYPE_STRING: + return UPB_TYPE_STRING; + case UPB_DESCRIPTOR_TYPE_BYTES: + return UPB_TYPE_BYTES; + case UPB_DESCRIPTOR_TYPE_GROUP: + case UPB_DESCRIPTOR_TYPE_MESSAGE: + return UPB_TYPE_MESSAGE; + } + UPB_UNREACHABLE(); +} + +upb_descriptortype_t upb_fielddef_descriptortype(const upb_fielddef *f) { + return f->type_; +} + +uint32_t upb_fielddef_index(const upb_fielddef *f) { + return f->index_; +} + +upb_label_t upb_fielddef_label(const upb_fielddef *f) { + return f->label_; +} + +uint32_t upb_fielddef_number(const upb_fielddef *f) { + return f->number_; +} + +bool upb_fielddef_isextension(const upb_fielddef *f) { + return f->is_extension_; +} + +bool upb_fielddef_lazy(const upb_fielddef *f) { + return f->lazy_; +} + +bool upb_fielddef_packed(const upb_fielddef *f) { + return f->packed_; +} + +const char *upb_fielddef_name(const upb_fielddef *f) { + return shortdefname(f->full_name); +} + +uint32_t upb_fielddef_selectorbase(const upb_fielddef *f) { + return f->selector_base; +} + +size_t upb_fielddef_getjsonname(const upb_fielddef *f, char *buf, size_t len) { + const char *name = upb_fielddef_name(f); + size_t src, dst = 0; + bool ucase_next = false; + +#define WRITE(byte) \ + ++dst; \ + if (dst < len) buf[dst - 1] = byte; \ + else if (dst == len) buf[dst - 1] = '\0' + + if (!name) { + WRITE('\0'); + return 0; + } + + /* Implement the transformation as described in the spec: + * 1. upper case all letters after an underscore. + * 2. remove all underscores. + */ + for (src = 0; name[src]; src++) { + if (name[src] == '_') { + ucase_next = true; + continue; + } + + if (ucase_next) { + WRITE(toupper(name[src])); + ucase_next = false; + } else { + WRITE(name[src]); + } + } + + WRITE('\0'); + return dst; + +#undef WRITE +} + +const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f) { + return f->msgdef; +} + +const upb_oneofdef *upb_fielddef_containingoneof(const upb_fielddef *f) { + return f->oneof; +} + +static void chkdefaulttype(const upb_fielddef *f, int ctype) { + UPB_UNUSED(f); + UPB_UNUSED(ctype); +} + +int64_t upb_fielddef_defaultint64(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_INT64); + return f->defaultval.sint; +} + +int32_t upb_fielddef_defaultint32(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_INT32); + return f->defaultval.sint; +} + +uint64_t upb_fielddef_defaultuint64(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_UINT64); + return f->defaultval.uint; +} + +uint32_t upb_fielddef_defaultuint32(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_UINT32); + return f->defaultval.uint; +} + +bool upb_fielddef_defaultbool(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_BOOL); + return f->defaultval.boolean; +} + +float upb_fielddef_defaultfloat(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_FLOAT); + return f->defaultval.flt; +} + +double upb_fielddef_defaultdouble(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_DOUBLE); + return f->defaultval.dbl; +} + +const char *upb_fielddef_defaultstr(const upb_fielddef *f, size_t *len) { + str_t *str = f->defaultval.str; + UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_STRING || + upb_fielddef_type(f) == UPB_TYPE_BYTES || + upb_fielddef_type(f) == UPB_TYPE_ENUM); + if (str) { + if (len) *len = str->len; + return str->str; + } else { + if (len) *len = 0; + return NULL; + } +} + +const upb_msgdef *upb_fielddef_msgsubdef(const upb_fielddef *f) { + UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_MESSAGE); + return f->sub.msgdef; +} + +const upb_enumdef *upb_fielddef_enumsubdef(const upb_fielddef *f) { + UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_ENUM); + return f->sub.enumdef; +} + +bool upb_fielddef_issubmsg(const upb_fielddef *f) { + return upb_fielddef_type(f) == UPB_TYPE_MESSAGE; +} + +bool upb_fielddef_isstring(const upb_fielddef *f) { + return upb_fielddef_type(f) == UPB_TYPE_STRING || + upb_fielddef_type(f) == UPB_TYPE_BYTES; +} + +bool upb_fielddef_isseq(const upb_fielddef *f) { + return upb_fielddef_label(f) == UPB_LABEL_REPEATED; +} + +bool upb_fielddef_isprimitive(const upb_fielddef *f) { + return !upb_fielddef_isstring(f) && !upb_fielddef_issubmsg(f); +} + +bool upb_fielddef_ismap(const upb_fielddef *f) { + return upb_fielddef_isseq(f) && upb_fielddef_issubmsg(f) && + upb_msgdef_mapentry(upb_fielddef_msgsubdef(f)); +} + +bool upb_fielddef_hassubdef(const upb_fielddef *f) { + return upb_fielddef_issubmsg(f) || upb_fielddef_type(f) == UPB_TYPE_ENUM; +} + +bool upb_fielddef_haspresence(const upb_fielddef *f) { + if (upb_fielddef_isseq(f)) return false; + if (upb_fielddef_issubmsg(f)) return true; + return f->file->syntax == UPB_SYNTAX_PROTO2; +} + +static bool between(int32_t x, int32_t low, int32_t high) { + return x >= low && x <= high; +} + +bool upb_fielddef_checklabel(int32_t label) { return between(label, 1, 3); } +bool upb_fielddef_checktype(int32_t type) { return between(type, 1, 11); } +bool upb_fielddef_checkintfmt(int32_t fmt) { return between(fmt, 1, 3); } + +bool upb_fielddef_checkdescriptortype(int32_t type) { + return between(type, 1, 18); +} + +/* upb_msgdef *****************************************************************/ + +const char *upb_msgdef_fullname(const upb_msgdef *m) { + return m->full_name; +} + +const upb_filedef *upb_msgdef_file(const upb_msgdef *m) { + return m->file; +} + +const char *upb_msgdef_name(const upb_msgdef *m) { + return shortdefname(m->full_name); +} + +upb_syntax_t upb_msgdef_syntax(const upb_msgdef *m) { + return m->file->syntax; +} + +size_t upb_msgdef_selectorcount(const upb_msgdef *m) { + return m->selector_count; +} + +uint32_t upb_msgdef_submsgfieldcount(const upb_msgdef *m) { + return m->submsg_field_count; +} + +const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i) { + upb_value val; + return upb_inttable_lookup32(&m->itof, i, &val) ? + upb_value_getconstptr(val) : NULL; +} + +const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name, + size_t len) { + upb_value val; + + if (!upb_strtable_lookup2(&m->ntof, name, len, &val)) { + return NULL; + } + + return unpack_def(val, UPB_DEFTYPE_FIELD); +} + +const upb_oneofdef *upb_msgdef_ntoo(const upb_msgdef *m, const char *name, + size_t len) { + upb_value val; + + if (!upb_strtable_lookup2(&m->ntof, name, len, &val)) { + return NULL; + } + + return unpack_def(val, UPB_DEFTYPE_ONEOF); +} + +bool upb_msgdef_lookupname(const upb_msgdef *m, const char *name, size_t len, + const upb_fielddef **f, const upb_oneofdef **o) { + upb_value val; + + if (!upb_strtable_lookup2(&m->ntof, name, len, &val)) { + return false; + } + + *o = unpack_def(val, UPB_DEFTYPE_ONEOF); + *f = unpack_def(val, UPB_DEFTYPE_FIELD); + UPB_ASSERT((*o != NULL) ^ (*f != NULL)); /* Exactly one of the two should be set. */ + return true; +} + +int upb_msgdef_numfields(const upb_msgdef *m) { + /* The number table contains only fields. */ + return upb_inttable_count(&m->itof); +} + +int upb_msgdef_numoneofs(const upb_msgdef *m) { + /* The name table includes oneofs, and the number table does not. */ + return upb_strtable_count(&m->ntof) - upb_inttable_count(&m->itof); +} + +bool upb_msgdef_mapentry(const upb_msgdef *m) { + return m->map_entry; +} + +upb_wellknowntype_t upb_msgdef_wellknowntype(const upb_msgdef *m) { + return m->well_known_type; +} + +bool upb_msgdef_isnumberwrapper(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type >= UPB_WELLKNOWN_DOUBLEVALUE && + type <= UPB_WELLKNOWN_UINT32VALUE; +} + +void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m) { + upb_inttable_begin(iter, &m->itof); +} + +void upb_msg_field_next(upb_msg_field_iter *iter) { upb_inttable_next(iter); } + +bool upb_msg_field_done(const upb_msg_field_iter *iter) { + return upb_inttable_done(iter); +} + +upb_fielddef *upb_msg_iter_field(const upb_msg_field_iter *iter) { + return (upb_fielddef *)upb_value_getconstptr(upb_inttable_iter_value(iter)); +} + +void upb_msg_field_iter_setdone(upb_msg_field_iter *iter) { + upb_inttable_iter_setdone(iter); +} + +bool upb_msg_field_iter_isequal(const upb_msg_field_iter * iter1, + const upb_msg_field_iter * iter2) { + return upb_inttable_iter_isequal(iter1, iter2); +} + +void upb_msg_oneof_begin(upb_msg_oneof_iter *iter, const upb_msgdef *m) { + upb_strtable_begin(iter, &m->ntof); + /* We need to skip past any initial fields. */ + while (!upb_strtable_done(iter) && + !unpack_def(upb_strtable_iter_value(iter), UPB_DEFTYPE_ONEOF)) { + upb_strtable_next(iter); + } +} + +void upb_msg_oneof_next(upb_msg_oneof_iter *iter) { + /* We need to skip past fields to return only oneofs. */ + do { + upb_strtable_next(iter); + } while (!upb_strtable_done(iter) && + !unpack_def(upb_strtable_iter_value(iter), UPB_DEFTYPE_ONEOF)); +} + +bool upb_msg_oneof_done(const upb_msg_oneof_iter *iter) { + return upb_strtable_done(iter); +} + +const upb_oneofdef *upb_msg_iter_oneof(const upb_msg_oneof_iter *iter) { + return unpack_def(upb_strtable_iter_value(iter), UPB_DEFTYPE_ONEOF); +} + +void upb_msg_oneof_iter_setdone(upb_msg_oneof_iter *iter) { + upb_strtable_iter_setdone(iter); +} + +bool upb_msg_oneof_iter_isequal(const upb_msg_oneof_iter *iter1, + const upb_msg_oneof_iter *iter2) { + return upb_strtable_iter_isequal(iter1, iter2); +} + +/* upb_oneofdef ***************************************************************/ + +const char *upb_oneofdef_name(const upb_oneofdef *o) { + return shortdefname(o->full_name); +} + +const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o) { + return o->parent; +} + +int upb_oneofdef_numfields(const upb_oneofdef *o) { + return upb_strtable_count(&o->ntof); +} + +uint32_t upb_oneofdef_index(const upb_oneofdef *o) { + return o->index; +} + +const upb_fielddef *upb_oneofdef_ntof(const upb_oneofdef *o, + const char *name, size_t length) { + upb_value val; + return upb_strtable_lookup2(&o->ntof, name, length, &val) ? + upb_value_getptr(val) : NULL; +} + +const upb_fielddef *upb_oneofdef_itof(const upb_oneofdef *o, uint32_t num) { + upb_value val; + return upb_inttable_lookup32(&o->itof, num, &val) ? + upb_value_getptr(val) : NULL; +} + +void upb_oneof_begin(upb_oneof_iter *iter, const upb_oneofdef *o) { + upb_inttable_begin(iter, &o->itof); +} + +void upb_oneof_next(upb_oneof_iter *iter) { + upb_inttable_next(iter); +} + +bool upb_oneof_done(upb_oneof_iter *iter) { + return upb_inttable_done(iter); +} + +upb_fielddef *upb_oneof_iter_field(const upb_oneof_iter *iter) { + return (upb_fielddef *)upb_value_getconstptr(upb_inttable_iter_value(iter)); +} + +void upb_oneof_iter_setdone(upb_oneof_iter *iter) { + upb_inttable_iter_setdone(iter); +} + +/* Code to build defs from descriptor protos. *********************************/ + +/* There is a question of how much validation to do here. It will be difficult + * to perfectly match the amount of validation performed by proto2. But since + * this code is used to directly build defs from Ruby (for example) we do need + * to validate important constraints like uniqueness of names and numbers. */ + +#define CHK(x) if (!(x)) { return false; } +#define CHK_OOM(x) if (!(x)) { upb_status_setoom(ctx->status); return false; } + +typedef struct { + const upb_symtab *symtab; + upb_filedef *file; /* File we are building. */ + upb_alloc *alloc; /* Allocate defs here. */ + upb_alloc *tmp; /* Alloc for addtab and any other tmp data. */ + upb_strtable *addtab; /* full_name -> packed def ptr for new defs. */ + upb_status *status; /* Record errors here. */ +} symtab_addctx; + +static char* strviewdup(const symtab_addctx *ctx, upb_strview view) { + return upb_strdup2(view.data, view.size, ctx->alloc); +} + +static bool streql2(const char *a, size_t n, const char *b) { + return n == strlen(b) && memcmp(a, b, n) == 0; +} + +static bool streql_view(upb_strview view, const char *b) { + return streql2(view.data, view.size, b); +} + +static const char *makefullname(const symtab_addctx *ctx, const char *prefix, + upb_strview name) { + if (prefix) { + /* ret = prefix + '.' + name; */ + size_t n = strlen(prefix); + char *ret = upb_malloc(ctx->alloc, n + name.size + 2); + CHK_OOM(ret); + strcpy(ret, prefix); + ret[n] = '.'; + memcpy(&ret[n + 1], name.data, name.size); + ret[n + 1 + name.size] = '\0'; + return ret; + } else { + return strviewdup(ctx, name); + } +} + +static bool symtab_add(const symtab_addctx *ctx, const char *name, + upb_value v) { + upb_value tmp; + if (upb_strtable_lookup(ctx->addtab, name, &tmp) || + upb_strtable_lookup(&ctx->symtab->syms, name, &tmp)) { + upb_status_seterrf(ctx->status, "duplicate symbol '%s'", name); + return false; + } + + CHK_OOM(upb_strtable_insert3(ctx->addtab, name, strlen(name), v, ctx->tmp)); + return true; +} + +/* Given a symbol and the base symbol inside which it is defined, find the + * symbol's definition in t. */ +static bool resolvename(const upb_strtable *t, const upb_fielddef *f, + const char *base, upb_strview sym, + upb_deftype_t type, upb_status *status, + const void **def) { + if(sym.size == 0) return NULL; + if(sym.data[0] == '.') { + /* Symbols starting with '.' are absolute, so we do a single lookup. + * Slice to omit the leading '.' */ + upb_value v; + if (!upb_strtable_lookup2(t, sym.data + 1, sym.size - 1, &v)) { + return false; + } + + *def = unpack_def(v, type); + + if (!*def) { + upb_status_seterrf(status, + "type mismatch when resolving field %s, name %s", + f->full_name, sym.data); + return false; + } + + return true; + } else { + /* Remove components from base until we find an entry or run out. + * TODO: This branch is totally broken, but currently not used. */ + (void)base; + UPB_ASSERT(false); + return false; + } +} + +const void *symtab_resolve(const symtab_addctx *ctx, const upb_fielddef *f, + const char *base, upb_strview sym, + upb_deftype_t type) { + const void *ret; + if (!resolvename(ctx->addtab, f, base, sym, type, ctx->status, &ret) && + !resolvename(&ctx->symtab->syms, f, base, sym, type, ctx->status, &ret)) { + if (upb_ok(ctx->status)) { + upb_status_seterrf(ctx->status, "couldn't resolve name '%s'", sym.data); + } + return false; + } + return ret; +} + +static bool create_oneofdef( + const symtab_addctx *ctx, upb_msgdef *m, + const google_protobuf_OneofDescriptorProto *oneof_proto) { + upb_oneofdef *o; + upb_strview name = google_protobuf_OneofDescriptorProto_name(oneof_proto); + upb_value v; + + o = (upb_oneofdef*)&m->oneofs[m->oneof_count++]; + o->parent = m; + o->full_name = makefullname(ctx, m->full_name, name); + + v = pack_def(o, UPB_DEFTYPE_ONEOF); + CHK_OOM(symtab_add(ctx, o->full_name, v)); + CHK_OOM(upb_strtable_insert3(&m->ntof, name.data, name.size, v, ctx->alloc)); + + CHK_OOM(upb_inttable_init2(&o->itof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + CHK_OOM(upb_strtable_init2(&o->ntof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + + return true; +} + +static bool parse_default(const symtab_addctx *ctx, const char *str, size_t len, + upb_fielddef *f) { + char *end; + char nullz[64]; + errno = 0; + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + /* Standard C number parsing functions expect null-terminated strings. */ + if (len >= sizeof(nullz) - 1) { + return false; + } + memcpy(nullz, str, len); + nullz[len] = '\0'; + str = nullz; + break; + default: + break; + } + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: { + long val = strtol(str, &end, 0); + CHK(val <= INT32_MAX && val >= INT32_MIN && errno != ERANGE && !*end); + f->defaultval.sint = val; + break; + } + case UPB_TYPE_ENUM: { + const upb_enumdef *e = f->sub.enumdef; + int32_t val; + CHK(upb_enumdef_ntoi(e, str, len, &val)); + f->defaultval.sint = val; + break; + } + case UPB_TYPE_INT64: { + /* XXX: Need to write our own strtoll, since it's not available in c89. */ + long long val = strtol(str, &end, 0); + CHK(val <= INT64_MAX && val >= INT64_MIN && errno != ERANGE && !*end); + f->defaultval.sint = val; + break; + } + case UPB_TYPE_UINT32: { + unsigned long val = strtoul(str, &end, 0); + CHK(val <= UINT32_MAX && errno != ERANGE && !*end); + f->defaultval.uint = val; + break; + } + case UPB_TYPE_UINT64: { + /* XXX: Need to write our own strtoull, since it's not available in c89. */ + unsigned long long val = strtoul(str, &end, 0); + CHK(val <= UINT64_MAX && errno != ERANGE && !*end); + f->defaultval.uint = val; + break; + } + case UPB_TYPE_DOUBLE: { + double val = strtod(str, &end); + CHK(errno != ERANGE && !*end); + f->defaultval.dbl = val; + break; + } + case UPB_TYPE_FLOAT: { + /* XXX: Need to write our own strtof, since it's not available in c89. */ + float val = strtod(str, &end); + CHK(errno != ERANGE && !*end); + f->defaultval.flt = val; + break; + } + case UPB_TYPE_BOOL: { + if (streql2(str, len, "false")) { + f->defaultval.boolean = false; + } else if (streql2(str, len, "true")) { + f->defaultval.boolean = true; + } else { + return false; + } + break; + } + case UPB_TYPE_STRING: + f->defaultval.str = newstr(ctx->alloc, str, len); + break; + case UPB_TYPE_BYTES: + /* XXX: need to interpret the C-escaped value. */ + f->defaultval.str = newstr(ctx->alloc, str, len); + break; + case UPB_TYPE_MESSAGE: + /* Should not have a default value. */ + return false; + } + return true; +} + +static void set_default_default(const symtab_addctx *ctx, upb_fielddef *f) { + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_ENUM: + f->defaultval.sint = 0; + break; + case UPB_TYPE_UINT64: + case UPB_TYPE_UINT32: + f->defaultval.uint = 0; + break; + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + f->defaultval.dbl = 0; + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + f->defaultval.str = newstr(ctx->alloc, NULL, 0); + break; + case UPB_TYPE_BOOL: + f->defaultval.boolean = false; + break; + case UPB_TYPE_MESSAGE: + break; + } +} + +static bool create_fielddef( + const symtab_addctx *ctx, const char *prefix, upb_msgdef *m, + const google_protobuf_FieldDescriptorProto *field_proto) { + upb_alloc *alloc = ctx->alloc; + upb_fielddef *f; + const google_protobuf_FieldOptions *options; + upb_strview name; + const char *full_name; + const char *shortname; + uint32_t field_number; + + if (!google_protobuf_FieldDescriptorProto_has_name(field_proto)) { + upb_status_seterrmsg(ctx->status, "field has no name"); + return false; + } + + name = google_protobuf_FieldDescriptorProto_name(field_proto); + CHK(upb_isident(name, false, ctx->status)); + full_name = makefullname(ctx, prefix, name); + shortname = shortdefname(full_name); + + field_number = google_protobuf_FieldDescriptorProto_number(field_proto); + + if (field_number == 0 || field_number > UPB_MAX_FIELDNUMBER) { + upb_status_seterrf(ctx->status, "invalid field number (%u)", field_number); + return false; + } + + if (m) { + /* direct message field. */ + upb_value v, packed_v; + + f = (upb_fielddef*)&m->fields[m->field_count++]; + f->msgdef = m; + f->is_extension_ = false; + + packed_v = pack_def(f, UPB_DEFTYPE_FIELD); + v = upb_value_constptr(f); + + if (!upb_strtable_insert3(&m->ntof, name.data, name.size, packed_v, alloc)) { + upb_status_seterrf(ctx->status, "duplicate field name (%s)", shortname); + return false; + } + + if (!upb_inttable_insert2(&m->itof, field_number, v, alloc)) { + upb_status_seterrf(ctx->status, "duplicate field number (%u)", + field_number); + return false; + } + } else { + /* extension field. */ + f = (upb_fielddef*)&ctx->file->exts[ctx->file->ext_count]; + f->is_extension_ = true; + CHK_OOM(symtab_add(ctx, full_name, pack_def(f, UPB_DEFTYPE_FIELD))); + } + + f->full_name = full_name; + f->file = ctx->file; + f->type_ = (int)google_protobuf_FieldDescriptorProto_type(field_proto); + f->label_ = (int)google_protobuf_FieldDescriptorProto_label(field_proto); + f->number_ = field_number; + f->oneof = NULL; + + /* We can't resolve the subdef or (in the case of extensions) the containing + * message yet, because it may not have been defined yet. We stash a pointer + * to the field_proto until later when we can properly resolve it. */ + f->sub.unresolved = field_proto; + + if (f->label_ == UPB_LABEL_REQUIRED && f->file->syntax == UPB_SYNTAX_PROTO3) { + upb_status_seterrf(ctx->status, "proto3 fields cannot be required (%s)", + f->full_name); + return false; + } + + if (google_protobuf_FieldDescriptorProto_has_oneof_index(field_proto)) { + int oneof_index = + google_protobuf_FieldDescriptorProto_oneof_index(field_proto); + upb_oneofdef *oneof; + upb_value v = upb_value_constptr(f); + + if (upb_fielddef_label(f) != UPB_LABEL_OPTIONAL) { + upb_status_seterrf(ctx->status, + "fields in oneof must have OPTIONAL label (%s)", + f->full_name); + return false; + } + + if (!m) { + upb_status_seterrf(ctx->status, + "oneof_index provided for extension field (%s)", + f->full_name); + return false; + } + + if (oneof_index >= m->oneof_count) { + upb_status_seterrf(ctx->status, "oneof_index out of range (%s)", + f->full_name); + return false; + } + + oneof = (upb_oneofdef*)&m->oneofs[oneof_index]; + f->oneof = oneof; + + CHK(upb_inttable_insert2(&oneof->itof, f->number_, v, alloc)); + CHK(upb_strtable_insert3(&oneof->ntof, name.data, name.size, v, alloc)); + } else { + f->oneof = NULL; + } + + if (google_protobuf_FieldDescriptorProto_has_options(field_proto)) { + options = google_protobuf_FieldDescriptorProto_options(field_proto); + f->lazy_ = google_protobuf_FieldOptions_lazy(options); + f->packed_ = google_protobuf_FieldOptions_packed(options); + } else { + f->lazy_ = false; + f->packed_ = false; + } + + return true; +} + +static bool create_enumdef( + const symtab_addctx *ctx, const char *prefix, + const google_protobuf_EnumDescriptorProto *enum_proto) { + upb_enumdef *e; + const google_protobuf_EnumValueDescriptorProto *const *values; + upb_strview name; + size_t i, n; + + name = google_protobuf_EnumDescriptorProto_name(enum_proto); + CHK(upb_isident(name, false, ctx->status)); + + e = (upb_enumdef*)&ctx->file->enums[ctx->file->enum_count++]; + e->full_name = makefullname(ctx, prefix, name); + CHK_OOM(symtab_add(ctx, e->full_name, pack_def(e, UPB_DEFTYPE_ENUM))); + + CHK_OOM(upb_strtable_init2(&e->ntoi, UPB_CTYPE_INT32, ctx->alloc)); + CHK_OOM(upb_inttable_init2(&e->iton, UPB_CTYPE_CSTR, ctx->alloc)); + + e->file = ctx->file; + e->defaultval = 0; + + values = google_protobuf_EnumDescriptorProto_value(enum_proto, &n); + + if (n == 0) { + upb_status_seterrf(ctx->status, + "enums must contain at least one value (%s)", + e->full_name); + return false; + } + + for (i = 0; i < n; i++) { + const google_protobuf_EnumValueDescriptorProto *value = values[i]; + upb_strview name = google_protobuf_EnumValueDescriptorProto_name(value); + char *name2 = strviewdup(ctx, name); + int32_t num = google_protobuf_EnumValueDescriptorProto_number(value); + upb_value v = upb_value_int32(num); + + if (i == 0 && e->file->syntax == UPB_SYNTAX_PROTO3 && num != 0) { + upb_status_seterrf(ctx->status, + "for proto3, the first enum value must be zero (%s)", + e->full_name); + return false; + } + + if (upb_strtable_lookup(&e->ntoi, name2, NULL)) { + upb_status_seterrf(ctx->status, "duplicate enum label '%s'", name2); + return false; + } + + CHK_OOM(name2) + CHK_OOM( + upb_strtable_insert3(&e->ntoi, name2, strlen(name2), v, ctx->alloc)); + + if (!upb_inttable_lookup(&e->iton, num, NULL)) { + upb_value v = upb_value_cstr(name2); + CHK_OOM(upb_inttable_insert2(&e->iton, num, v, ctx->alloc)); + } + } + + upb_inttable_compact2(&e->iton, ctx->alloc); + + return true; +} + +static bool create_msgdef(const symtab_addctx *ctx, const char *prefix, + const google_protobuf_DescriptorProto *msg_proto) { + upb_msgdef *m; + const google_protobuf_MessageOptions *options; + const google_protobuf_OneofDescriptorProto *const *oneofs; + const google_protobuf_FieldDescriptorProto *const *fields; + const google_protobuf_EnumDescriptorProto *const *enums; + const google_protobuf_DescriptorProto *const *msgs; + size_t i, n; + upb_strview name; + + name = google_protobuf_DescriptorProto_name(msg_proto); + CHK(upb_isident(name, false, ctx->status)); + + m = (upb_msgdef*)&ctx->file->msgs[ctx->file->msg_count++]; + m->full_name = makefullname(ctx, prefix, name); + CHK_OOM(symtab_add(ctx, m->full_name, pack_def(m, UPB_DEFTYPE_MSG))); + + CHK_OOM(upb_inttable_init2(&m->itof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + CHK_OOM(upb_strtable_init2(&m->ntof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + + m->file = ctx->file; + m->map_entry = false; + + options = google_protobuf_DescriptorProto_options(msg_proto); + + if (options) { + m->map_entry = google_protobuf_MessageOptions_map_entry(options); + } + + oneofs = google_protobuf_DescriptorProto_oneof_decl(msg_proto, &n); + m->oneof_count = 0; + m->oneofs = upb_malloc(ctx->alloc, sizeof(*m->oneofs) * n); + for (i = 0; i < n; i++) { + CHK(create_oneofdef(ctx, m, oneofs[i])); + } + + fields = google_protobuf_DescriptorProto_field(msg_proto, &n); + m->field_count = 0; + m->fields = upb_malloc(ctx->alloc, sizeof(*m->fields) * n); + for (i = 0; i < n; i++) { + CHK(create_fielddef(ctx, m->full_name, m, fields[i])); + } + + CHK(assign_msg_indices(m, ctx->status)); + assign_msg_wellknowntype(m); + upb_inttable_compact2(&m->itof, ctx->alloc); + + /* This message is built. Now build nested messages and enums. */ + + enums = google_protobuf_DescriptorProto_enum_type(msg_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_enumdef(ctx, m->full_name, enums[i])); + } + + msgs = google_protobuf_DescriptorProto_nested_type(msg_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_msgdef(ctx, m->full_name, msgs[i])); + } + + return true; +} + +typedef struct { + int msg_count; + int enum_count; + int ext_count; +} decl_counts; + +static void count_types_in_msg(const google_protobuf_DescriptorProto *msg_proto, + decl_counts *counts) { + const google_protobuf_DescriptorProto *const *msgs; + size_t i, n; + + counts->msg_count++; + + msgs = google_protobuf_DescriptorProto_nested_type(msg_proto, &n); + for (i = 0; i < n; i++) { + count_types_in_msg(msgs[i], counts); + } + + google_protobuf_DescriptorProto_enum_type(msg_proto, &n); + counts->enum_count += n; + + google_protobuf_DescriptorProto_extension(msg_proto, &n); + counts->ext_count += n; +} + +static void count_types_in_file( + const google_protobuf_FileDescriptorProto *file_proto, + decl_counts *counts) { + const google_protobuf_DescriptorProto *const *msgs; + size_t i, n; + + msgs = google_protobuf_FileDescriptorProto_message_type(file_proto, &n); + for (i = 0; i < n; i++) { + count_types_in_msg(msgs[i], counts); + } + + google_protobuf_FileDescriptorProto_enum_type(file_proto, &n); + counts->enum_count += n; + + google_protobuf_FileDescriptorProto_extension(file_proto, &n); + counts->ext_count += n; +} + +static bool resolve_fielddef(const symtab_addctx *ctx, const char *prefix, + upb_fielddef *f) { + upb_strview name; + const google_protobuf_FieldDescriptorProto *field_proto = f->sub.unresolved; + + if (f->is_extension_) { + if (!google_protobuf_FieldDescriptorProto_has_extendee(field_proto)) { + upb_status_seterrf(ctx->status, + "extension for field '%s' had no extendee", + f->full_name); + return false; + } + + name = google_protobuf_FieldDescriptorProto_extendee(field_proto); + f->msgdef = symtab_resolve(ctx, f, prefix, name, UPB_DEFTYPE_MSG); + CHK(f->msgdef); + } + + if ((upb_fielddef_issubmsg(f) || f->type_ == UPB_DESCRIPTOR_TYPE_ENUM) && + !google_protobuf_FieldDescriptorProto_has_type_name(field_proto)) { + upb_status_seterrf(ctx->status, "field '%s' is missing type name", + f->full_name); + return false; + } + + name = google_protobuf_FieldDescriptorProto_type_name(field_proto); + + if (upb_fielddef_issubmsg(f)) { + f->sub.msgdef = symtab_resolve(ctx, f, prefix, name, UPB_DEFTYPE_MSG); + CHK(f->sub.msgdef); + } else if (f->type_ == UPB_DESCRIPTOR_TYPE_ENUM) { + f->sub.enumdef = symtab_resolve(ctx, f, prefix, name, UPB_DEFTYPE_ENUM); + CHK(f->sub.enumdef); + } + + /* Have to delay resolving of the default value until now because of the enum + * case, since enum defaults are specified with a label. */ + if (google_protobuf_FieldDescriptorProto_has_default_value(field_proto)) { + upb_strview defaultval = + google_protobuf_FieldDescriptorProto_default_value(field_proto); + + if (f->file->syntax == UPB_SYNTAX_PROTO3) { + upb_status_seterrf(ctx->status, + "proto3 fields cannot have explicit defaults (%s)", + f->full_name); + return false; + } + + if (upb_fielddef_issubmsg(f)) { + upb_status_seterrf(ctx->status, + "message fields cannot have explicit defaults (%s)", + f->full_name); + return false; + } + + if (!parse_default(ctx, defaultval.data, defaultval.size, f)) { + upb_status_seterrf(ctx->status, + "couldn't parse default '" UPB_STRVIEW_FORMAT + "' for field (%s)", + UPB_STRVIEW_ARGS(defaultval), f->full_name); + return false; + } + } else { + set_default_default(ctx, f); + } + + return true; +} + +static bool build_filedef( + const symtab_addctx *ctx, upb_filedef *file, + const google_protobuf_FileDescriptorProto *file_proto) { + upb_alloc *alloc = ctx->alloc; + const google_protobuf_FileOptions *file_options_proto; + const google_protobuf_DescriptorProto *const *msgs; + const google_protobuf_EnumDescriptorProto *const *enums; + const google_protobuf_FieldDescriptorProto *const *exts; + const upb_strview* strs; + size_t i, n; + decl_counts counts = {0}; + + count_types_in_file(file_proto, &counts); + + file->msgs = upb_malloc(alloc, sizeof(*file->msgs) * counts.msg_count); + file->enums = upb_malloc(alloc, sizeof(*file->enums) * counts.enum_count); + file->exts = upb_malloc(alloc, sizeof(*file->exts) * counts.ext_count); + + CHK_OOM(counts.msg_count == 0 || file->msgs); + CHK_OOM(counts.enum_count == 0 || file->enums); + CHK_OOM(counts.ext_count == 0 || file->exts); + + /* We increment these as defs are added. */ + file->msg_count = 0; + file->enum_count = 0; + file->ext_count = 0; + + if (!google_protobuf_FileDescriptorProto_has_name(file_proto)) { + upb_status_seterrmsg(ctx->status, "File has no name"); + return false; + } + + file->name = + strviewdup(ctx, google_protobuf_FileDescriptorProto_name(file_proto)); + file->phpprefix = NULL; + file->phpnamespace = NULL; + + if (google_protobuf_FileDescriptorProto_has_package(file_proto)) { + upb_strview package = + google_protobuf_FileDescriptorProto_package(file_proto); + CHK(upb_isident(package, true, ctx->status)); + file->package = strviewdup(ctx, package); + } else { + file->package = NULL; + } + + if (google_protobuf_FileDescriptorProto_has_syntax(file_proto)) { + upb_strview syntax = + google_protobuf_FileDescriptorProto_syntax(file_proto); + + if (streql_view(syntax, "proto2")) { + file->syntax = UPB_SYNTAX_PROTO2; + } else if (streql_view(syntax, "proto3")) { + file->syntax = UPB_SYNTAX_PROTO3; + } else { + upb_status_seterrf(ctx->status, "Invalid syntax '%s'", syntax); + return false; + } + } else { + file->syntax = UPB_SYNTAX_PROTO2; + } + + /* Read options. */ + file_options_proto = google_protobuf_FileDescriptorProto_options(file_proto); + if (file_options_proto) { + if (google_protobuf_FileOptions_has_php_class_prefix(file_options_proto)) { + file->phpprefix = strviewdup( + ctx, + google_protobuf_FileOptions_php_class_prefix(file_options_proto)); + } + if (google_protobuf_FileOptions_has_php_namespace(file_options_proto)) { + file->phpnamespace = strviewdup( + ctx, google_protobuf_FileOptions_php_namespace(file_options_proto)); + } + } + + /* Verify dependencies. */ + strs = google_protobuf_FileDescriptorProto_dependency(file_proto, &n); + file->deps = upb_malloc(alloc, sizeof(*file->deps) * n) ; + CHK_OOM(n == 0 || file->deps); + + for (i = 0; i < n; i++) { + upb_strview dep_name = strs[i]; + upb_value v; + if (!upb_strtable_lookup2(&ctx->symtab->files, dep_name.data, + dep_name.size, &v)) { + upb_status_seterrf(ctx->status, + "Depends on file '" UPB_STRVIEW_FORMAT + "', but it has not been loaded", + UPB_STRVIEW_ARGS(dep_name)); + return false; + } + file->deps[i] = upb_value_getconstptr(v); + } + + /* Create messages. */ + msgs = google_protobuf_FileDescriptorProto_message_type(file_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_msgdef(ctx, file->package, msgs[i])); + } + + /* Create enums. */ + enums = google_protobuf_FileDescriptorProto_enum_type(file_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_enumdef(ctx, file->package, enums[i])); + } + + /* Create extensions. */ + exts = google_protobuf_FileDescriptorProto_extension(file_proto, &n); + file->exts = upb_malloc(alloc, sizeof(*file->exts) * n); + CHK_OOM(n == 0 || file->exts); + for (i = 0; i < n; i++) { + CHK(create_fielddef(ctx, file->package, NULL, exts[i])); + } + + /* Now that all names are in the table, resolve references. */ + for (i = 0; i < file->ext_count; i++) { + CHK(resolve_fielddef(ctx, file->package, (upb_fielddef*)&file->exts[i])); + } + + for (i = 0; i < file->msg_count; i++) { + const upb_msgdef *m = &file->msgs[i]; + int j; + for (j = 0; j < m->field_count; j++) { + CHK(resolve_fielddef(ctx, m->full_name, (upb_fielddef*)&m->fields[j])); + } + } + + return true; + } + +static bool upb_symtab_addtotabs(upb_symtab *s, symtab_addctx *ctx, + upb_status *status) { + const upb_filedef *file = ctx->file; + upb_alloc *alloc = upb_arena_alloc(s->arena); + upb_strtable_iter iter; + + CHK_OOM(upb_strtable_insert3(&s->files, file->name, strlen(file->name), + upb_value_constptr(file), alloc)); + + upb_strtable_begin(&iter, ctx->addtab); + for (; !upb_strtable_done(&iter); upb_strtable_next(&iter)) { + const char *key = upb_strtable_iter_key(&iter); + size_t keylen = upb_strtable_iter_keylength(&iter); + upb_value value = upb_strtable_iter_value(&iter); + CHK_OOM(upb_strtable_insert3(&s->syms, key, keylen, value, alloc)); + } + + return true; +} + +/* upb_filedef ****************************************************************/ + +const char *upb_filedef_name(const upb_filedef *f) { + return f->name; +} + +const char *upb_filedef_package(const upb_filedef *f) { + return f->package; +} + +const char *upb_filedef_phpprefix(const upb_filedef *f) { + return f->phpprefix; +} + +const char *upb_filedef_phpnamespace(const upb_filedef *f) { + return f->phpnamespace; +} + +upb_syntax_t upb_filedef_syntax(const upb_filedef *f) { + return f->syntax; +} + +int upb_filedef_msgcount(const upb_filedef *f) { + return f->msg_count; +} + +int upb_filedef_depcount(const upb_filedef *f) { + return f->dep_count; +} + +int upb_filedef_enumcount(const upb_filedef *f) { + return f->enum_count; +} + +const upb_filedef *upb_filedef_dep(const upb_filedef *f, int i) { + return i < 0 || i >= f->dep_count ? NULL : f->deps[i]; +} + +const upb_msgdef *upb_filedef_msg(const upb_filedef *f, int i) { + return i < 0 || i >= f->msg_count ? NULL : &f->msgs[i]; +} + +const upb_enumdef *upb_filedef_enum(const upb_filedef *f, int i) { + return i < 0 || i >= f->enum_count ? NULL : &f->enums[i]; +} + +void upb_symtab_free(upb_symtab *s) { + upb_arena_free(s->arena); + upb_gfree(s); +} + +upb_symtab *upb_symtab_new(void) { + upb_symtab *s = upb_gmalloc(sizeof(*s)); + upb_alloc *alloc; + + if (!s) { + return NULL; + } + + s->arena = upb_arena_new(); + alloc = upb_arena_alloc(s->arena); + + if (!upb_strtable_init2(&s->syms, UPB_CTYPE_CONSTPTR, alloc) || + !upb_strtable_init2(&s->files, UPB_CTYPE_CONSTPTR, alloc)) { + upb_arena_free(s->arena); + upb_gfree(s); + s = NULL; + } + return s; +} + +const upb_msgdef *upb_symtab_lookupmsg(const upb_symtab *s, const char *sym) { + upb_value v; + return upb_strtable_lookup(&s->syms, sym, &v) ? + unpack_def(v, UPB_DEFTYPE_MSG) : NULL; +} + +const upb_msgdef *upb_symtab_lookupmsg2(const upb_symtab *s, const char *sym, + size_t len) { + upb_value v; + return upb_strtable_lookup2(&s->syms, sym, len, &v) ? + unpack_def(v, UPB_DEFTYPE_MSG) : NULL; +} + +const upb_enumdef *upb_symtab_lookupenum(const upb_symtab *s, const char *sym) { + upb_value v; + return upb_strtable_lookup(&s->syms, sym, &v) ? + unpack_def(v, UPB_DEFTYPE_ENUM) : NULL; +} + +const upb_filedef *upb_symtab_lookupfile(const upb_symtab *s, const char *name) { + upb_value v; + return upb_strtable_lookup(&s->files, name, &v) ? upb_value_getconstptr(v) + : NULL; +} + +const upb_filedef *upb_symtab_addfile( + upb_symtab *s, const google_protobuf_FileDescriptorProto *file_proto, + upb_status *status) { + upb_arena *tmparena = upb_arena_new(); + upb_strtable addtab; + upb_alloc *alloc = upb_arena_alloc(s->arena); + upb_filedef *file = upb_malloc(alloc, sizeof(*file)); + bool ok; + symtab_addctx ctx; + + ctx.file = file; + ctx.symtab = s; + ctx.alloc = alloc; + ctx.tmp = upb_arena_alloc(tmparena); + ctx.addtab = &addtab; + ctx.status = status; + + ok = file && + upb_strtable_init2(&addtab, UPB_CTYPE_CONSTPTR, ctx.tmp) && + build_filedef(&ctx, file, file_proto) && + upb_symtab_addtotabs(s, &ctx, status); + + upb_arena_free(tmparena); + return ok ? file : NULL; +} + +/* Include here since we want most of this file to be stdio-free. */ +#include + +bool _upb_symtab_loaddefinit(upb_symtab *s, const upb_def_init *init) { + /* Since this function should never fail (it would indicate a bug in upb) we + * print errors to stderr instead of returning error status to the user. */ + upb_def_init **deps = init->deps; + google_protobuf_FileDescriptorProto *file; + upb_arena *arena; + upb_status status; + + upb_status_clear(&status); + + if (upb_strtable_lookup(&s->files, init->filename, NULL)) { + return true; + } + + arena = upb_arena_new(); + + for (; *deps; deps++) { + if (!_upb_symtab_loaddefinit(s, *deps)) goto err; + } + + file = google_protobuf_FileDescriptorProto_parse( + init->descriptor.data, init->descriptor.size, arena); + + if (!file) { + upb_status_seterrf( + &status, + "Failed to parse compiled-in descriptor for file '%s'. This should " + "never happen.", + init->filename); + goto err; + } + + if (!upb_symtab_addfile(s, file, &status)) goto err; + + upb_arena_free(arena); + return true; + +err: + fprintf(stderr, "Error loading compiled-in descriptor: %s\n", + upb_status_errmsg(&status)); + upb_arena_free(arena); + return false; +} + +#undef CHK +#undef CHK_OOM diff --git a/third_party/upb/upb/def.h b/third_party/upb/upb/def.h new file mode 100644 index 00000000000..9be285794ee --- /dev/null +++ b/third_party/upb/upb/def.h @@ -0,0 +1,909 @@ +/* +** Defs are upb's internal representation of the constructs that can appear +** in a .proto file: +** +** - upb::MessageDefPtr (upb_msgdef): describes a "message" construct. +** - upb::FieldDefPtr (upb_fielddef): describes a message field. +** - upb::FileDefPtr (upb_filedef): describes a .proto file and its defs. +** - upb::EnumDefPtr (upb_enumdef): describes an enum. +** - upb::OneofDefPtr (upb_oneofdef): describes a oneof. +** +** TODO: definitions of services. +** +** This is a mixed C/C++ interface that offers a full API to both languages. +** See the top-level README for more information. +*/ + +#ifndef UPB_DEF_H_ +#define UPB_DEF_H_ + +#include "upb/upb.h" +#include "upb/table.int.h" +#include "google/protobuf/descriptor.upb.h" + +#ifdef __cplusplus +#include +#include +#include +#include + +namespace upb { +class EnumDefPtr; +class FieldDefPtr; +class FileDefPtr; +class MessageDefPtr; +class OneofDefPtr; +class SymbolTable; +} +#endif + +#include "upb/port_def.inc" + +struct upb_enumdef; +typedef struct upb_enumdef upb_enumdef; +struct upb_fielddef; +typedef struct upb_fielddef upb_fielddef; +struct upb_filedef; +typedef struct upb_filedef upb_filedef; +struct upb_msgdef; +typedef struct upb_msgdef upb_msgdef; +struct upb_oneofdef; +typedef struct upb_oneofdef upb_oneofdef; +struct upb_symtab; +typedef struct upb_symtab upb_symtab; + +typedef enum { + UPB_SYNTAX_PROTO2 = 2, + UPB_SYNTAX_PROTO3 = 3 +} upb_syntax_t; + +/* All the different kind of well known type messages. For simplicity of check, + * number wrappers and string wrappers are grouped together. Make sure the + * order and merber of these groups are not changed. + */ +typedef enum { + UPB_WELLKNOWN_UNSPECIFIED, + UPB_WELLKNOWN_ANY, + UPB_WELLKNOWN_FIELDMASK, + UPB_WELLKNOWN_DURATION, + UPB_WELLKNOWN_TIMESTAMP, + /* number wrappers */ + UPB_WELLKNOWN_DOUBLEVALUE, + UPB_WELLKNOWN_FLOATVALUE, + UPB_WELLKNOWN_INT64VALUE, + UPB_WELLKNOWN_UINT64VALUE, + UPB_WELLKNOWN_INT32VALUE, + UPB_WELLKNOWN_UINT32VALUE, + /* string wrappers */ + UPB_WELLKNOWN_STRINGVALUE, + UPB_WELLKNOWN_BYTESVALUE, + UPB_WELLKNOWN_BOOLVALUE, + UPB_WELLKNOWN_VALUE, + UPB_WELLKNOWN_LISTVALUE, + UPB_WELLKNOWN_STRUCT +} upb_wellknowntype_t; + +/* upb_fielddef ***************************************************************/ + +/* Maximum field number allowed for FieldDefs. This is an inherent limit of the + * protobuf wire format. */ +#define UPB_MAX_FIELDNUMBER ((1 << 29) - 1) + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_fielddef_fullname(const upb_fielddef *f); +upb_fieldtype_t upb_fielddef_type(const upb_fielddef *f); +upb_descriptortype_t upb_fielddef_descriptortype(const upb_fielddef *f); +upb_label_t upb_fielddef_label(const upb_fielddef *f); +uint32_t upb_fielddef_number(const upb_fielddef *f); +const char *upb_fielddef_name(const upb_fielddef *f); +bool upb_fielddef_isextension(const upb_fielddef *f); +bool upb_fielddef_lazy(const upb_fielddef *f); +bool upb_fielddef_packed(const upb_fielddef *f); +size_t upb_fielddef_getjsonname(const upb_fielddef *f, char *buf, size_t len); +const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f); +const upb_oneofdef *upb_fielddef_containingoneof(const upb_fielddef *f); +uint32_t upb_fielddef_index(const upb_fielddef *f); +bool upb_fielddef_issubmsg(const upb_fielddef *f); +bool upb_fielddef_isstring(const upb_fielddef *f); +bool upb_fielddef_isseq(const upb_fielddef *f); +bool upb_fielddef_isprimitive(const upb_fielddef *f); +bool upb_fielddef_ismap(const upb_fielddef *f); +int64_t upb_fielddef_defaultint64(const upb_fielddef *f); +int32_t upb_fielddef_defaultint32(const upb_fielddef *f); +uint64_t upb_fielddef_defaultuint64(const upb_fielddef *f); +uint32_t upb_fielddef_defaultuint32(const upb_fielddef *f); +bool upb_fielddef_defaultbool(const upb_fielddef *f); +float upb_fielddef_defaultfloat(const upb_fielddef *f); +double upb_fielddef_defaultdouble(const upb_fielddef *f); +const char *upb_fielddef_defaultstr(const upb_fielddef *f, size_t *len); +bool upb_fielddef_hassubdef(const upb_fielddef *f); +bool upb_fielddef_haspresence(const upb_fielddef *f); +const upb_msgdef *upb_fielddef_msgsubdef(const upb_fielddef *f); +const upb_enumdef *upb_fielddef_enumsubdef(const upb_fielddef *f); + +/* Internal only. */ +uint32_t upb_fielddef_selectorbase(const upb_fielddef *f); + +#ifdef __cplusplus +} /* extern "C" */ + +/* A upb_fielddef describes a single field in a message. It is most often + * found as a part of a upb_msgdef, but can also stand alone to represent + * an extension. */ +class upb::FieldDefPtr { + public: + FieldDefPtr() : ptr_(nullptr) {} + explicit FieldDefPtr(const upb_fielddef *ptr) : ptr_(ptr) {} + + const upb_fielddef* ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + typedef upb_fieldtype_t Type; + typedef upb_label_t Label; + typedef upb_descriptortype_t DescriptorType; + + const char* full_name() const { return upb_fielddef_fullname(ptr_); } + + Type type() const { return upb_fielddef_type(ptr_); } + Label label() const { return upb_fielddef_label(ptr_); } + const char* name() const { return upb_fielddef_name(ptr_); } + uint32_t number() const { return upb_fielddef_number(ptr_); } + bool is_extension() const { return upb_fielddef_isextension(ptr_); } + + /* Copies the JSON name for this field into the given buffer. Returns the + * actual size of the JSON name, including the NULL terminator. If the + * return value is 0, the JSON name is unset. If the return value is + * greater than len, the JSON name was truncated. The buffer is always + * NULL-terminated if len > 0. + * + * The JSON name always defaults to a camelCased version of the regular + * name. However if the regular name is unset, the JSON name will be unset + * also. + */ + size_t GetJsonName(char *buf, size_t len) const { + return upb_fielddef_getjsonname(ptr_, buf, len); + } + + /* Convenience version of the above function which copies the JSON name + * into the given string, returning false if the name is not set. */ + template + bool GetJsonName(T* str) { + str->resize(GetJsonName(NULL, 0)); + GetJsonName(&(*str)[0], str->size()); + return str->size() > 0; + } + + /* For UPB_TYPE_MESSAGE fields only where is_tag_delimited() == false, + * indicates whether this field should have lazy parsing handlers that yield + * the unparsed string for the submessage. + * + * TODO(haberman): I think we want to move this into a FieldOptions container + * when we add support for custom options (the FieldOptions struct will + * contain both regular FieldOptions like "lazy" *and* custom options). */ + bool lazy() const { return upb_fielddef_lazy(ptr_); } + + /* For non-string, non-submessage fields, this indicates whether binary + * protobufs are encoded in packed or non-packed format. + * + * TODO(haberman): see note above about putting options like this into a + * FieldOptions container. */ + bool packed() const { return upb_fielddef_packed(ptr_); } + + /* An integer that can be used as an index into an array of fields for + * whatever message this field belongs to. Guaranteed to be less than + * f->containing_type()->field_count(). May only be accessed once the def has + * been finalized. */ + uint32_t index() const { return upb_fielddef_index(ptr_); } + + /* The MessageDef to which this field belongs. + * + * If this field has been added to a MessageDef, that message can be retrieved + * directly (this is always the case for frozen FieldDefs). + * + * If the field has not yet been added to a MessageDef, you can set the name + * of the containing type symbolically instead. This is mostly useful for + * extensions, where the extension is declared separately from the message. */ + MessageDefPtr containing_type() const; + + /* The OneofDef to which this field belongs, or NULL if this field is not part + * of a oneof. */ + OneofDefPtr containing_oneof() const; + + /* The field's type according to the enum in descriptor.proto. This is not + * the same as UPB_TYPE_*, because it distinguishes between (for example) + * INT32 and SINT32, whereas our "type" enum does not. This return of + * descriptor_type() is a function of type(), integer_format(), and + * is_tag_delimited(). */ + DescriptorType descriptor_type() const { + return upb_fielddef_descriptortype(ptr_); + } + + /* Convenient field type tests. */ + bool IsSubMessage() const { return upb_fielddef_issubmsg(ptr_); } + bool IsString() const { return upb_fielddef_isstring(ptr_); } + bool IsSequence() const { return upb_fielddef_isseq(ptr_); } + bool IsPrimitive() const { return upb_fielddef_isprimitive(ptr_); } + bool IsMap() const { return upb_fielddef_ismap(ptr_); } + + /* Returns the non-string default value for this fielddef, which may either + * be something the client set explicitly or the "default default" (0 for + * numbers, empty for strings). The field's type indicates the type of the + * returned value, except for enum fields that are still mutable. + * + * Requires that the given function matches the field's current type. */ + int64_t default_int64() const { return upb_fielddef_defaultint64(ptr_); } + int32_t default_int32() const { return upb_fielddef_defaultint32(ptr_); } + uint64_t default_uint64() const { return upb_fielddef_defaultuint64(ptr_); } + uint32_t default_uint32() const { return upb_fielddef_defaultuint32(ptr_); } + bool default_bool() const { return upb_fielddef_defaultbool(ptr_); } + float default_float() const { return upb_fielddef_defaultfloat(ptr_); } + double default_double() const { return upb_fielddef_defaultdouble(ptr_); } + + /* The resulting string is always NULL-terminated. If non-NULL, the length + * will be stored in *len. */ + const char *default_string(size_t * len) const { + return upb_fielddef_defaultstr(ptr_, len); + } + + /* Returns the enum or submessage def for this field, if any. The field's + * type must match (ie. you may only call enum_subdef() for fields where + * type() == UPB_TYPE_ENUM). */ + EnumDefPtr enum_subdef() const; + MessageDefPtr message_subdef() const; + + private: + const upb_fielddef *ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_oneofdef ***************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef upb_inttable_iter upb_oneof_iter; + +const char *upb_oneofdef_name(const upb_oneofdef *o); +const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o); +int upb_oneofdef_numfields(const upb_oneofdef *o); +uint32_t upb_oneofdef_index(const upb_oneofdef *o); + +/* Oneof lookups: + * - ntof: look up a field by name. + * - ntofz: look up a field by name (as a null-terminated string). + * - itof: look up a field by number. */ +const upb_fielddef *upb_oneofdef_ntof(const upb_oneofdef *o, + const char *name, size_t length); +UPB_INLINE const upb_fielddef *upb_oneofdef_ntofz(const upb_oneofdef *o, + const char *name) { + return upb_oneofdef_ntof(o, name, strlen(name)); +} +const upb_fielddef *upb_oneofdef_itof(const upb_oneofdef *o, uint32_t num); + +/* upb_oneof_iter i; + * for(upb_oneof_begin(&i, e); !upb_oneof_done(&i); upb_oneof_next(&i)) { + * // ... + * } + */ +void upb_oneof_begin(upb_oneof_iter *iter, const upb_oneofdef *o); +void upb_oneof_next(upb_oneof_iter *iter); +bool upb_oneof_done(upb_oneof_iter *iter); +upb_fielddef *upb_oneof_iter_field(const upb_oneof_iter *iter); +void upb_oneof_iter_setdone(upb_oneof_iter *iter); +bool upb_oneof_iter_isequal(const upb_oneof_iter *iter1, + const upb_oneof_iter *iter2); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Class that represents a oneof. */ +class upb::OneofDefPtr { + public: + OneofDefPtr() : ptr_(nullptr) {} + explicit OneofDefPtr(const upb_oneofdef *ptr) : ptr_(ptr) {} + + const upb_oneofdef* ptr() const { return ptr_; } + explicit operator bool() { return ptr_ != nullptr; } + + /* Returns the MessageDef that owns this OneofDef. */ + MessageDefPtr containing_type() const; + + /* Returns the name of this oneof. This is the name used to look up the oneof + * by name once added to a message def. */ + const char* name() const { return upb_oneofdef_name(ptr_); } + + /* Returns the number of fields currently defined in the oneof. */ + int field_count() const { return upb_oneofdef_numfields(ptr_); } + + /* Looks up by name. */ + FieldDefPtr FindFieldByName(const char *name, size_t len) const { + return FieldDefPtr(upb_oneofdef_ntof(ptr_, name, len)); + } + FieldDefPtr FindFieldByName(const char* name) const { + return FieldDefPtr(upb_oneofdef_ntofz(ptr_, name)); + } + + template + FieldDefPtr FindFieldByName(const T& str) const { + return FindFieldByName(str.c_str(), str.size()); + } + + /* Looks up by tag number. */ + FieldDefPtr FindFieldByNumber(uint32_t num) const { + return FieldDefPtr(upb_oneofdef_itof(ptr_, num)); + } + + class const_iterator + : public std::iterator { + public: + void operator++() { upb_oneof_next(&iter_); } + + FieldDefPtr operator*() const { + return FieldDefPtr(upb_oneof_iter_field(&iter_)); + } + + bool operator!=(const const_iterator& other) const { + return !upb_oneof_iter_isequal(&iter_, &other.iter_); + } + + bool operator==(const const_iterator& other) const { + return upb_oneof_iter_isequal(&iter_, &other.iter_); + } + + private: + friend class OneofDefPtr; + + const_iterator() {} + explicit const_iterator(OneofDefPtr o) { + upb_oneof_begin(&iter_, o.ptr()); + } + static const_iterator end() { + const_iterator iter; + upb_oneof_iter_setdone(&iter.iter_); + return iter; + } + + upb_oneof_iter iter_; + }; + + const_iterator begin() const { return const_iterator(*this); } + const_iterator end() const { return const_iterator::end(); } + + private: + const upb_oneofdef *ptr_; +}; + +inline upb::OneofDefPtr upb::FieldDefPtr::containing_oneof() const { + return OneofDefPtr(upb_fielddef_containingoneof(ptr_)); +} + +#endif /* __cplusplus */ + +/* upb_msgdef *****************************************************************/ + +typedef upb_inttable_iter upb_msg_field_iter; +typedef upb_strtable_iter upb_msg_oneof_iter; + +/* Well-known field tag numbers for map-entry messages. */ +#define UPB_MAPENTRY_KEY 1 +#define UPB_MAPENTRY_VALUE 2 + +/* Well-known field tag numbers for Any messages. */ +#define UPB_ANY_TYPE 1 +#define UPB_ANY_VALUE 2 + +/* Well-known field tag numbers for timestamp messages. */ +#define UPB_DURATION_SECONDS 1 +#define UPB_DURATION_NANOS 2 + +/* Well-known field tag numbers for duration messages. */ +#define UPB_TIMESTAMP_SECONDS 1 +#define UPB_TIMESTAMP_NANOS 2 + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_msgdef_fullname(const upb_msgdef *m); +const upb_filedef *upb_msgdef_file(const upb_msgdef *m); +const char *upb_msgdef_name(const upb_msgdef *m); +int upb_msgdef_numoneofs(const upb_msgdef *m); +upb_syntax_t upb_msgdef_syntax(const upb_msgdef *m); +bool upb_msgdef_mapentry(const upb_msgdef *m); +upb_wellknowntype_t upb_msgdef_wellknowntype(const upb_msgdef *m); +bool upb_msgdef_isnumberwrapper(const upb_msgdef *m); +bool upb_msgdef_setsyntax(upb_msgdef *m, upb_syntax_t syntax); +const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i); +const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name, + size_t len); +const upb_oneofdef *upb_msgdef_ntoo(const upb_msgdef *m, const char *name, + size_t len); +int upb_msgdef_numfields(const upb_msgdef *m); +int upb_msgdef_numoneofs(const upb_msgdef *m); + +UPB_INLINE const upb_oneofdef *upb_msgdef_ntooz(const upb_msgdef *m, + const char *name) { + return upb_msgdef_ntoo(m, name, strlen(name)); +} + +UPB_INLINE const upb_fielddef *upb_msgdef_ntofz(const upb_msgdef *m, + const char *name) { + return upb_msgdef_ntof(m, name, strlen(name)); +} + +/* Internal-only. */ +size_t upb_msgdef_selectorcount(const upb_msgdef *m); +uint32_t upb_msgdef_submsgfieldcount(const upb_msgdef *m); + +/* Lookup of either field or oneof by name. Returns whether either was found. + * If the return is true, then the found def will be set, and the non-found + * one set to NULL. */ +bool upb_msgdef_lookupname(const upb_msgdef *m, const char *name, size_t len, + const upb_fielddef **f, const upb_oneofdef **o); + +UPB_INLINE bool upb_msgdef_lookupnamez(const upb_msgdef *m, const char *name, + const upb_fielddef **f, + const upb_oneofdef **o) { + return upb_msgdef_lookupname(m, name, strlen(name), f, o); +} + +/* Iteration over fields and oneofs. For example: + * + * upb_msg_field_iter i; + * for(upb_msg_field_begin(&i, m); + * !upb_msg_field_done(&i); + * upb_msg_field_next(&i)) { + * upb_fielddef *f = upb_msg_iter_field(&i); + * // ... + * } + * + * For C we don't have separate iterators for const and non-const. + * It is the caller's responsibility to cast the upb_fielddef* to + * const if the upb_msgdef* is const. */ +void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m); +void upb_msg_field_next(upb_msg_field_iter *iter); +bool upb_msg_field_done(const upb_msg_field_iter *iter); +upb_fielddef *upb_msg_iter_field(const upb_msg_field_iter *iter); +void upb_msg_field_iter_setdone(upb_msg_field_iter *iter); +bool upb_msg_field_iter_isequal(const upb_msg_field_iter * iter1, + const upb_msg_field_iter * iter2); + +/* Similar to above, we also support iterating through the oneofs in a + * msgdef. */ +void upb_msg_oneof_begin(upb_msg_oneof_iter * iter, const upb_msgdef *m); +void upb_msg_oneof_next(upb_msg_oneof_iter * iter); +bool upb_msg_oneof_done(const upb_msg_oneof_iter *iter); +const upb_oneofdef *upb_msg_iter_oneof(const upb_msg_oneof_iter *iter); +void upb_msg_oneof_iter_setdone(upb_msg_oneof_iter * iter); +bool upb_msg_oneof_iter_isequal(const upb_msg_oneof_iter *iter1, + const upb_msg_oneof_iter *iter2); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Structure that describes a single .proto message type. */ +class upb::MessageDefPtr { + public: + MessageDefPtr() : ptr_(nullptr) {} + explicit MessageDefPtr(const upb_msgdef *ptr) : ptr_(ptr) {} + + const upb_msgdef *ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + const char* full_name() const { return upb_msgdef_fullname(ptr_); } + const char* name() const { return upb_msgdef_name(ptr_); } + + /* The number of fields that belong to the MessageDef. */ + int field_count() const { return upb_msgdef_numfields(ptr_); } + + /* The number of oneofs that belong to the MessageDef. */ + int oneof_count() const { return upb_msgdef_numoneofs(ptr_); } + + upb_syntax_t syntax() const { return upb_msgdef_syntax(ptr_); } + + /* These return null pointers if the field is not found. */ + FieldDefPtr FindFieldByNumber(uint32_t number) const { + return FieldDefPtr(upb_msgdef_itof(ptr_, number)); + } + FieldDefPtr FindFieldByName(const char* name, size_t len) const { + return FieldDefPtr(upb_msgdef_ntof(ptr_, name, len)); + } + FieldDefPtr FindFieldByName(const char *name) const { + return FieldDefPtr(upb_msgdef_ntofz(ptr_, name)); + } + + template + FieldDefPtr FindFieldByName(const T& str) const { + return FindFieldByName(str.c_str(), str.size()); + } + + OneofDefPtr FindOneofByName(const char* name, size_t len) const { + return OneofDefPtr(upb_msgdef_ntoo(ptr_, name, len)); + } + + OneofDefPtr FindOneofByName(const char *name) const { + return OneofDefPtr(upb_msgdef_ntooz(ptr_, name)); + } + + template + OneofDefPtr FindOneofByName(const T &str) const { + return FindOneofByName(str.c_str(), str.size()); + } + + /* Is this message a map entry? */ + bool mapentry() const { return upb_msgdef_mapentry(ptr_); } + + /* Return the type of well known type message. UPB_WELLKNOWN_UNSPECIFIED for + * non-well-known message. */ + upb_wellknowntype_t wellknowntype() const { + return upb_msgdef_wellknowntype(ptr_); + } + + /* Whether is a number wrapper. */ + bool isnumberwrapper() const { return upb_msgdef_isnumberwrapper(ptr_); } + + /* Iteration over fields. The order is undefined. */ + class const_field_iterator + : public std::iterator { + public: + void operator++() { upb_msg_field_next(&iter_); } + + FieldDefPtr operator*() const { + return FieldDefPtr(upb_msg_iter_field(&iter_)); + } + + bool operator!=(const const_field_iterator &other) const { + return !upb_msg_field_iter_isequal(&iter_, &other.iter_); + } + + bool operator==(const const_field_iterator &other) const { + return upb_msg_field_iter_isequal(&iter_, &other.iter_); + } + + private: + friend class MessageDefPtr; + + explicit const_field_iterator() {} + + explicit const_field_iterator(MessageDefPtr msg) { + upb_msg_field_begin(&iter_, msg.ptr()); + } + + static const_field_iterator end() { + const_field_iterator iter; + upb_msg_field_iter_setdone(&iter.iter_); + return iter; + } + + upb_msg_field_iter iter_; + }; + + /* Iteration over oneofs. The order is undefined. */ + class const_oneof_iterator + : public std::iterator { + public: + + void operator++() { upb_msg_oneof_next(&iter_); } + + OneofDefPtr operator*() const { + return OneofDefPtr(upb_msg_iter_oneof(&iter_)); + } + + bool operator!=(const const_oneof_iterator& other) const { + return !upb_msg_oneof_iter_isequal(&iter_, &other.iter_); + } + + bool operator==(const const_oneof_iterator &other) const { + return upb_msg_oneof_iter_isequal(&iter_, &other.iter_); + } + + private: + friend class MessageDefPtr; + + const_oneof_iterator() {} + + explicit const_oneof_iterator(MessageDefPtr msg) { + upb_msg_oneof_begin(&iter_, msg.ptr()); + } + + static const_oneof_iterator end() { + const_oneof_iterator iter; + upb_msg_oneof_iter_setdone(&iter.iter_); + return iter; + } + + upb_msg_oneof_iter iter_; + }; + + class ConstFieldAccessor { + public: + explicit ConstFieldAccessor(const upb_msgdef* md) : md_(md) {} + const_field_iterator begin() { return MessageDefPtr(md_).field_begin(); } + const_field_iterator end() { return MessageDefPtr(md_).field_end(); } + private: + const upb_msgdef* md_; + }; + + class ConstOneofAccessor { + public: + explicit ConstOneofAccessor(const upb_msgdef* md) : md_(md) {} + const_oneof_iterator begin() { return MessageDefPtr(md_).oneof_begin(); } + const_oneof_iterator end() { return MessageDefPtr(md_).oneof_end(); } + private: + const upb_msgdef* md_; + }; + + const_field_iterator field_begin() const { + return const_field_iterator(*this); + } + + const_field_iterator field_end() const { return const_field_iterator::end(); } + + const_oneof_iterator oneof_begin() const { + return const_oneof_iterator(*this); + } + + const_oneof_iterator oneof_end() const { return const_oneof_iterator::end(); } + + ConstFieldAccessor fields() const { return ConstFieldAccessor(ptr()); } + ConstOneofAccessor oneofs() const { return ConstOneofAccessor(ptr()); } + + private: + const upb_msgdef* ptr_; +}; + +inline upb::MessageDefPtr upb::FieldDefPtr::message_subdef() const { + return MessageDefPtr(upb_fielddef_msgsubdef(ptr_)); +} + +inline upb::MessageDefPtr upb::FieldDefPtr::containing_type() const { + return MessageDefPtr(upb_fielddef_containingtype(ptr_)); +} + +inline upb::MessageDefPtr upb::OneofDefPtr::containing_type() const { + return MessageDefPtr(upb_oneofdef_containingtype(ptr_)); +} + +#endif /* __cplusplus */ + +/* upb_enumdef ****************************************************************/ + +typedef upb_strtable_iter upb_enum_iter; + +const char *upb_enumdef_fullname(const upb_enumdef *e); +const char *upb_enumdef_name(const upb_enumdef *e); +const upb_filedef *upb_enumdef_file(const upb_enumdef *e); +int32_t upb_enumdef_default(const upb_enumdef *e); +int upb_enumdef_numvals(const upb_enumdef *e); + +/* Enum lookups: + * - ntoi: look up a name with specified length. + * - ntoiz: look up a name provided as a null-terminated string. + * - iton: look up an integer, returning the name as a null-terminated + * string. */ +bool upb_enumdef_ntoi(const upb_enumdef *e, const char *name, size_t len, + int32_t *num); +UPB_INLINE bool upb_enumdef_ntoiz(const upb_enumdef *e, + const char *name, int32_t *num) { + return upb_enumdef_ntoi(e, name, strlen(name), num); +} +const char *upb_enumdef_iton(const upb_enumdef *e, int32_t num); + +/* upb_enum_iter i; + * for(upb_enum_begin(&i, e); !upb_enum_done(&i); upb_enum_next(&i)) { + * // ... + * } + */ +void upb_enum_begin(upb_enum_iter *iter, const upb_enumdef *e); +void upb_enum_next(upb_enum_iter *iter); +bool upb_enum_done(upb_enum_iter *iter); +const char *upb_enum_iter_name(upb_enum_iter *iter); +int32_t upb_enum_iter_number(upb_enum_iter *iter); + +#ifdef __cplusplus + +class upb::EnumDefPtr { + public: + EnumDefPtr() : ptr_(nullptr) {} + explicit EnumDefPtr(const upb_enumdef* ptr) : ptr_(ptr) {} + + const upb_enumdef* ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + const char* full_name() const { return upb_enumdef_fullname(ptr_); } + const char* name() const { return upb_enumdef_name(ptr_); } + + /* The value that is used as the default when no field default is specified. + * If not set explicitly, the first value that was added will be used. + * The default value must be a member of the enum. + * Requires that value_count() > 0. */ + int32_t default_value() const { return upb_enumdef_default(ptr_); } + + /* Returns the number of values currently defined in the enum. Note that + * multiple names can refer to the same number, so this may be greater than + * the total number of unique numbers. */ + int value_count() const { return upb_enumdef_numvals(ptr_); } + + /* Lookups from name to integer, returning true if found. */ + bool FindValueByName(const char *name, int32_t *num) const { + return upb_enumdef_ntoiz(ptr_, name, num); + } + + /* Finds the name corresponding to the given number, or NULL if none was + * found. If more than one name corresponds to this number, returns the + * first one that was added. */ + const char *FindValueByNumber(int32_t num) const { + return upb_enumdef_iton(ptr_, num); + } + + /* Iteration over name/value pairs. The order is undefined. + * Adding an enum val invalidates any iterators. + * + * TODO: make compatible with range-for, with elements as pairs? */ + class Iterator { + public: + explicit Iterator(EnumDefPtr e) { upb_enum_begin(&iter_, e.ptr()); } + + int32_t number() { return upb_enum_iter_number(&iter_); } + const char *name() { return upb_enum_iter_name(&iter_); } + bool Done() { return upb_enum_done(&iter_); } + void Next() { return upb_enum_next(&iter_); } + + private: + upb_enum_iter iter_; + }; + + private: + const upb_enumdef *ptr_; +}; + +inline upb::EnumDefPtr upb::FieldDefPtr::enum_subdef() const { + return EnumDefPtr(upb_fielddef_enumsubdef(ptr_)); +} + +#endif /* __cplusplus */ + +/* upb_filedef ****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_filedef_name(const upb_filedef *f); +const char *upb_filedef_package(const upb_filedef *f); +const char *upb_filedef_phpprefix(const upb_filedef *f); +const char *upb_filedef_phpnamespace(const upb_filedef *f); +upb_syntax_t upb_filedef_syntax(const upb_filedef *f); +int upb_filedef_depcount(const upb_filedef *f); +int upb_filedef_msgcount(const upb_filedef *f); +int upb_filedef_enumcount(const upb_filedef *f); +const upb_filedef *upb_filedef_dep(const upb_filedef *f, int i); +const upb_msgdef *upb_filedef_msg(const upb_filedef *f, int i); +const upb_enumdef *upb_filedef_enum(const upb_filedef *f, int i); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Class that represents a .proto file with some things defined in it. + * + * Many users won't care about FileDefs, but they are necessary if you want to + * read the values of file-level options. */ +class upb::FileDefPtr { + public: + explicit FileDefPtr(const upb_filedef *ptr) : ptr_(ptr) {} + + const upb_filedef* ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + /* Get/set name of the file (eg. "foo/bar.proto"). */ + const char* name() const { return upb_filedef_name(ptr_); } + + /* Package name for definitions inside the file (eg. "foo.bar"). */ + const char* package() const { return upb_filedef_package(ptr_); } + + /* Sets the php class prefix which is prepended to all php generated classes + * from this .proto. Default is empty. */ + const char* phpprefix() const { return upb_filedef_phpprefix(ptr_); } + + /* Use this option to change the namespace of php generated classes. Default + * is empty. When this option is empty, the package name will be used for + * determining the namespace. */ + const char* phpnamespace() const { return upb_filedef_phpnamespace(ptr_); } + + /* Syntax for the file. Defaults to proto2. */ + upb_syntax_t syntax() const { return upb_filedef_syntax(ptr_); } + + /* Get the list of dependencies from the file. These are returned in the + * order that they were added to the FileDefPtr. */ + int dependency_count() const { return upb_filedef_depcount(ptr_); } + const FileDefPtr dependency(int index) const { + return FileDefPtr(upb_filedef_dep(ptr_, index)); + } + + private: + const upb_filedef* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_symtab *****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +upb_symtab *upb_symtab_new(void); +void upb_symtab_free(upb_symtab* s); +const upb_msgdef *upb_symtab_lookupmsg(const upb_symtab *s, const char *sym); +const upb_msgdef *upb_symtab_lookupmsg2( + const upb_symtab *s, const char *sym, size_t len); +const upb_enumdef *upb_symtab_lookupenum(const upb_symtab *s, const char *sym); +const upb_filedef *upb_symtab_lookupfile(const upb_symtab *s, const char *name); +int upb_symtab_filecount(const upb_symtab *s); +const upb_filedef *upb_symtab_addfile( + upb_symtab *s, const google_protobuf_FileDescriptorProto *file, + upb_status *status); + +/* For generated code only: loads a generated descriptor. */ +typedef struct upb_def_init { + struct upb_def_init **deps; + const char *filename; + upb_strview descriptor; +} upb_def_init; + +bool _upb_symtab_loaddefinit(upb_symtab *s, const upb_def_init *init); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Non-const methods in upb::SymbolTable are NOT thread-safe. */ +class upb::SymbolTable { + public: + SymbolTable() : ptr_(upb_symtab_new(), upb_symtab_free) {} + explicit SymbolTable(upb_symtab* s) : ptr_(s, upb_symtab_free) {} + + const upb_symtab* ptr() const { return ptr_.get(); } + upb_symtab* ptr() { return ptr_.get(); } + + /* Finds an entry in the symbol table with this exact name. If not found, + * returns NULL. */ + MessageDefPtr LookupMessage(const char *sym) const { + return MessageDefPtr(upb_symtab_lookupmsg(ptr_.get(), sym)); + } + + EnumDefPtr LookupEnum(const char *sym) const { + return EnumDefPtr(upb_symtab_lookupenum(ptr_.get(), sym)); + } + + FileDefPtr LookupFile(const char *name) const { + return FileDefPtr(upb_symtab_lookupfile(ptr_.get(), name)); + } + + /* TODO: iteration? */ + + /* Adds the given serialized FileDescriptorProto to the pool. */ + FileDefPtr AddFile(const google_protobuf_FileDescriptorProto *file_proto, + Status *status) { + return FileDefPtr( + upb_symtab_addfile(ptr_.get(), file_proto, status->ptr())); + } + + private: + std::unique_ptr ptr_; +}; + +UPB_INLINE const char* upb_safecstr(const std::string& str) { + UPB_ASSERT(str.size() == std::strlen(str.c_str())); + return str.c_str(); +} + +#endif /* __cplusplus */ + +#include "upb/port_undef.inc" + +#endif /* UPB_DEF_H_ */ diff --git a/third_party/upb/upb/encode.c b/third_party/upb/upb/encode.c new file mode 100644 index 00000000000..43d24cdbfcc --- /dev/null +++ b/third_party/upb/upb/encode.c @@ -0,0 +1,378 @@ +/* We encode backwards, to avoid pre-computing lengths (one-pass encode). */ + +#include "upb/encode.h" + +#include + +#include "upb/msg.h" +#include "upb/upb.h" + +#include "upb/port_def.inc" + +#define UPB_PB_VARINT_MAX_LEN 10 +#define CHK(x) do { if (!(x)) { return false; } } while(0) + +static size_t upb_encode_varint(uint64_t val, char *buf) { + size_t i; + if (val < 128) { buf[0] = val; return 1; } + i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +static uint32_t upb_zzencode_32(int32_t n) { return ((uint32_t)n << 1) ^ (n >> 31); } +static uint64_t upb_zzencode_64(int64_t n) { return ((uint64_t)n << 1) ^ (n >> 63); } + +typedef struct { + upb_alloc *alloc; + char *buf, *ptr, *limit; +} upb_encstate; + +static size_t upb_roundup_pow2(size_t bytes) { + size_t ret = 128; + while (ret < bytes) { + ret *= 2; + } + return ret; +} + +static bool upb_encode_growbuffer(upb_encstate *e, size_t bytes) { + size_t old_size = e->limit - e->buf; + size_t new_size = upb_roundup_pow2(bytes + (e->limit - e->ptr)); + char *new_buf = upb_realloc(e->alloc, e->buf, old_size, new_size); + CHK(new_buf); + + /* We want previous data at the end, realloc() put it at the beginning. */ + if (old_size > 0) { + memmove(new_buf + new_size - old_size, e->buf, old_size); + } + + e->ptr = new_buf + new_size - (e->limit - e->ptr); + e->limit = new_buf + new_size; + e->buf = new_buf; + return true; +} + +/* Call to ensure that at least "bytes" bytes are available for writing at + * e->ptr. Returns false if the bytes could not be allocated. */ +static bool upb_encode_reserve(upb_encstate *e, size_t bytes) { + CHK(UPB_LIKELY((size_t)(e->ptr - e->buf) >= bytes) || + upb_encode_growbuffer(e, bytes)); + + e->ptr -= bytes; + return true; +} + +/* Writes the given bytes to the buffer, handling reserve/advance. */ +static bool upb_put_bytes(upb_encstate *e, const void *data, size_t len) { + CHK(upb_encode_reserve(e, len)); + memcpy(e->ptr, data, len); + return true; +} + +static bool upb_put_fixed64(upb_encstate *e, uint64_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return upb_put_bytes(e, &val, sizeof(uint64_t)); +} + +static bool upb_put_fixed32(upb_encstate *e, uint32_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return upb_put_bytes(e, &val, sizeof(uint32_t)); +} + +static bool upb_put_varint(upb_encstate *e, uint64_t val) { + size_t len; + char *start; + CHK(upb_encode_reserve(e, UPB_PB_VARINT_MAX_LEN)); + len = upb_encode_varint(val, e->ptr); + start = e->ptr + UPB_PB_VARINT_MAX_LEN - len; + memmove(start, e->ptr, len); + e->ptr = start; + return true; +} + +static bool upb_put_double(upb_encstate *e, double d) { + uint64_t u64; + UPB_ASSERT(sizeof(double) == sizeof(uint64_t)); + memcpy(&u64, &d, sizeof(uint64_t)); + return upb_put_fixed64(e, u64); +} + +static bool upb_put_float(upb_encstate *e, float d) { + uint32_t u32; + UPB_ASSERT(sizeof(float) == sizeof(uint32_t)); + memcpy(&u32, &d, sizeof(uint32_t)); + return upb_put_fixed32(e, u32); +} + +static uint32_t upb_readcase(const char *msg, const upb_msglayout_field *f) { + uint32_t ret; + uint32_t offset = ~f->presence; + memcpy(&ret, msg + offset, sizeof(ret)); + return ret; +} + +static bool upb_readhasbit(const char *msg, const upb_msglayout_field *f) { + uint32_t hasbit = f->presence; + UPB_ASSERT(f->presence > 0); + return msg[hasbit / 8] & (1 << (hasbit % 8)); +} + +static bool upb_put_tag(upb_encstate *e, int field_number, int wire_type) { + return upb_put_varint(e, (field_number << 3) | wire_type); +} + +static bool upb_put_fixedarray(upb_encstate *e, const upb_array *arr, + size_t size) { + size_t bytes = arr->len * size; + return upb_put_bytes(e, arr->data, bytes) && upb_put_varint(e, bytes); +} + +bool upb_encode_message(upb_encstate *e, const char *msg, + const upb_msglayout *m, size_t *size); + +static bool upb_encode_array(upb_encstate *e, const char *field_mem, + const upb_msglayout *m, + const upb_msglayout_field *f) { + const upb_array *arr = *(const upb_array**)field_mem; + + if (arr == NULL || arr->len == 0) { + return true; + } + +#define VARINT_CASE(ctype, encode) { \ + ctype *start = arr->data; \ + ctype *ptr = start + arr->len; \ + size_t pre_len = e->limit - e->ptr; \ + do { \ + ptr--; \ + CHK(upb_put_varint(e, encode)); \ + } while (ptr != start); \ + CHK(upb_put_varint(e, e->limit - e->ptr - pre_len)); \ +} \ +break; \ +do { ; } while(0) + + switch (f->descriptortype) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + CHK(upb_put_fixedarray(e, arr, sizeof(double))); + break; + case UPB_DESCRIPTOR_TYPE_FLOAT: + CHK(upb_put_fixedarray(e, arr, sizeof(float))); + break; + case UPB_DESCRIPTOR_TYPE_SFIXED64: + case UPB_DESCRIPTOR_TYPE_FIXED64: + CHK(upb_put_fixedarray(e, arr, sizeof(uint64_t))); + break; + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + CHK(upb_put_fixedarray(e, arr, sizeof(uint32_t))); + break; + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + VARINT_CASE(uint64_t, *ptr); + case UPB_DESCRIPTOR_TYPE_UINT32: + VARINT_CASE(uint32_t, *ptr); + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_ENUM: + VARINT_CASE(int32_t, (int64_t)*ptr); + case UPB_DESCRIPTOR_TYPE_BOOL: + VARINT_CASE(bool, *ptr); + case UPB_DESCRIPTOR_TYPE_SINT32: + VARINT_CASE(int32_t, upb_zzencode_32(*ptr)); + case UPB_DESCRIPTOR_TYPE_SINT64: + VARINT_CASE(int64_t, upb_zzencode_64(*ptr)); + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview *start = arr->data; + upb_strview *ptr = start + arr->len; + do { + ptr--; + CHK(upb_put_bytes(e, ptr->data, ptr->size) && + upb_put_varint(e, ptr->size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED)); + } while (ptr != start); + return true; + } + case UPB_DESCRIPTOR_TYPE_GROUP: { + void **start = arr->data; + void **ptr = start + arr->len; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + do { + size_t size; + ptr--; + CHK(upb_put_tag(e, f->number, UPB_WIRE_TYPE_END_GROUP) && + upb_encode_message(e, *ptr, subm, &size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_START_GROUP)); + } while (ptr != start); + return true; + } + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + void **start = arr->data; + void **ptr = start + arr->len; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + do { + size_t size; + ptr--; + CHK(upb_encode_message(e, *ptr, subm, &size) && + upb_put_varint(e, size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED)); + } while (ptr != start); + return true; + } + } +#undef VARINT_CASE + + /* We encode all primitive arrays as packed, regardless of what was specified + * in the .proto file. Could special case 1-sized arrays. */ + CHK(upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED)); + return true; +} + +static bool upb_encode_scalarfield(upb_encstate *e, const char *field_mem, + const upb_msglayout *m, + const upb_msglayout_field *f, + bool skip_zero_value) { +#define CASE(ctype, type, wire_type, encodeval) do { \ + ctype val = *(ctype*)field_mem; \ + if (skip_zero_value && val == 0) { \ + return true; \ + } \ + return upb_put_ ## type(e, encodeval) && \ + upb_put_tag(e, f->number, wire_type); \ +} while(0) + + switch (f->descriptortype) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + CASE(double, double, UPB_WIRE_TYPE_64BIT, val); + case UPB_DESCRIPTOR_TYPE_FLOAT: + CASE(float, float, UPB_WIRE_TYPE_32BIT, val); + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + CASE(uint64_t, varint, UPB_WIRE_TYPE_VARINT, val); + case UPB_DESCRIPTOR_TYPE_UINT32: + CASE(uint32_t, varint, UPB_WIRE_TYPE_VARINT, val); + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_ENUM: + CASE(int32_t, varint, UPB_WIRE_TYPE_VARINT, (int64_t)val); + case UPB_DESCRIPTOR_TYPE_SFIXED64: + case UPB_DESCRIPTOR_TYPE_FIXED64: + CASE(uint64_t, fixed64, UPB_WIRE_TYPE_64BIT, val); + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + CASE(uint32_t, fixed32, UPB_WIRE_TYPE_32BIT, val); + case UPB_DESCRIPTOR_TYPE_BOOL: + CASE(bool, varint, UPB_WIRE_TYPE_VARINT, val); + case UPB_DESCRIPTOR_TYPE_SINT32: + CASE(int32_t, varint, UPB_WIRE_TYPE_VARINT, upb_zzencode_32(val)); + case UPB_DESCRIPTOR_TYPE_SINT64: + CASE(int64_t, varint, UPB_WIRE_TYPE_VARINT, upb_zzencode_64(val)); + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview view = *(upb_strview*)field_mem; + if (skip_zero_value && view.size == 0) { + return true; + } + return upb_put_bytes(e, view.data, view.size) && + upb_put_varint(e, view.size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED); + } + case UPB_DESCRIPTOR_TYPE_GROUP: { + size_t size; + void *submsg = *(void **)field_mem; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + if (submsg == NULL) { + return true; + } + return upb_put_tag(e, f->number, UPB_WIRE_TYPE_END_GROUP) && + upb_encode_message(e, submsg, subm, &size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_START_GROUP); + } + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + size_t size; + void *submsg = *(void **)field_mem; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + if (submsg == NULL) { + return true; + } + return upb_encode_message(e, submsg, subm, &size) && + upb_put_varint(e, size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED); + } + } +#undef CASE + UPB_UNREACHABLE(); +} + +bool upb_encode_message(upb_encstate *e, const char *msg, + const upb_msglayout *m, size_t *size) { + int i; + size_t pre_len = e->limit - e->ptr; + const char *unknown; + size_t unknown_size; + + for (i = m->field_count - 1; i >= 0; i--) { + const upb_msglayout_field *f = &m->fields[i]; + + if (f->label == UPB_LABEL_REPEATED) { + CHK(upb_encode_array(e, msg + f->offset, m, f)); + } else { + bool skip_empty = false; + if (f->presence == 0) { + /* Proto3 presence. */ + skip_empty = true; + } else if (f->presence > 0) { + /* Proto2 presence: hasbit. */ + if (!upb_readhasbit(msg, f)) { + continue; + } + } else { + /* Field is in a oneof. */ + if (upb_readcase(msg, f) != f->number) { + continue; + } + } + CHK(upb_encode_scalarfield(e, msg + f->offset, m, f, skip_empty)); + } + } + + unknown = upb_msg_getunknown(msg, &unknown_size); + + if (unknown) { + upb_put_bytes(e, unknown, unknown_size); + } + + *size = (e->limit - e->ptr) - pre_len; + return true; +} + +char *upb_encode(const void *msg, const upb_msglayout *m, upb_arena *arena, + size_t *size) { + upb_encstate e; + e.alloc = upb_arena_alloc(arena); + e.buf = NULL; + e.limit = NULL; + e.ptr = NULL; + + if (!upb_encode_message(&e, msg, m, size)) { + *size = 0; + return NULL; + } + + *size = e.limit - e.ptr; + + if (*size == 0) { + static char ch; + return &ch; + } else { + UPB_ASSERT(e.ptr); + return e.ptr; + } +} + +#undef CHK diff --git a/third_party/upb/upb/encode.h b/third_party/upb/upb/encode.h new file mode 100644 index 00000000000..68427770585 --- /dev/null +++ b/third_party/upb/upb/encode.h @@ -0,0 +1,21 @@ +/* +** upb_encode: parsing into a upb_msg using a upb_msglayout. +*/ + +#ifndef UPB_ENCODE_H_ +#define UPB_ENCODE_H_ + +#include "upb/msg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +char *upb_encode(const void *msg, const upb_msglayout *l, upb_arena *arena, + size_t *size); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_ENCODE_H_ */ diff --git a/third_party/upb/upb/generated_util.h b/third_party/upb/upb/generated_util.h new file mode 100644 index 00000000000..234bcdad3c9 --- /dev/null +++ b/third_party/upb/upb/generated_util.h @@ -0,0 +1,105 @@ +/* +** Functions for use by generated code. These are not public and users must +** not call them directly. +*/ + +#ifndef UPB_GENERATED_UTIL_H_ +#define UPB_GENERATED_UTIL_H_ + +#include +#include "upb/msg.h" + +#include "upb/port_def.inc" + +#define PTR_AT(msg, ofs, type) (type*)((const char*)msg + ofs) + +UPB_INLINE const void *_upb_array_accessor(const void *msg, size_t ofs, + size_t *size) { + const upb_array *arr = *PTR_AT(msg, ofs, const upb_array*); + if (arr) { + if (size) *size = arr->len; + return arr->data; + } else { + if (size) *size = 0; + return NULL; + } +} + +UPB_INLINE void *_upb_array_mutable_accessor(void *msg, size_t ofs, + size_t *size) { + upb_array *arr = *PTR_AT(msg, ofs, upb_array*); + if (arr) { + if (size) *size = arr->len; + return arr->data; + } else { + if (size) *size = 0; + return NULL; + } +} + +/* TODO(haberman): this is a mess. It will improve when upb_array no longer + * carries reflective state (type, elem_size). */ +UPB_INLINE void *_upb_array_resize_accessor(void *msg, size_t ofs, size_t size, + size_t elem_size, + upb_fieldtype_t type, + upb_arena *arena) { + upb_array *arr = *PTR_AT(msg, ofs, upb_array*); + + if (!arr) { + arr = upb_array_new(arena); + if (!arr) return NULL; + *PTR_AT(msg, ofs, upb_array*) = arr; + } + + if (size > arr->size) { + size_t new_size = UPB_MAX(arr->size, 4); + size_t old_bytes = arr->size * elem_size; + size_t new_bytes; + while (new_size < size) new_size *= 2; + new_bytes = new_size * elem_size; + arr->data = upb_arena_realloc(arena, arr->data, old_bytes, new_bytes); + if (!arr->data) { + return NULL; + } + arr->size = new_size; + } + + arr->len = size; + return arr->data; +} + +UPB_INLINE bool _upb_array_append_accessor(void *msg, size_t ofs, + size_t elem_size, + upb_fieldtype_t type, + const void *value, + upb_arena *arena) { + upb_array *arr = *PTR_AT(msg, ofs, upb_array*); + size_t i = arr ? arr->len : 0; + void *data = + _upb_array_resize_accessor(msg, ofs, i + 1, elem_size, type, arena); + if (!data) return false; + memcpy(PTR_AT(data, i * elem_size, char), value, elem_size); + return true; +} + +UPB_INLINE bool _upb_has_field(const void *msg, size_t idx) { + return (*PTR_AT(msg, idx / 8, const char) & (1 << (idx % 8))) != 0; +} + +UPB_INLINE bool _upb_sethas(const void *msg, size_t idx) { + return (*PTR_AT(msg, idx / 8, char)) |= (char)(1 << (idx % 8)); +} + +UPB_INLINE bool _upb_clearhas(const void *msg, size_t idx) { + return (*PTR_AT(msg, idx / 8, char)) &= (char)(~(1 << (idx % 8))); +} + +UPB_INLINE bool _upb_has_oneof_field(const void *msg, size_t case_ofs, int32_t num) { + return *PTR_AT(msg, case_ofs, int32_t) == num; +} + +#undef PTR_AT + +#include "upb/port_undef.inc" + +#endif /* UPB_GENERATED_UTIL_H_ */ diff --git a/third_party/upb/upb/handlers-inl.h b/third_party/upb/upb/handlers-inl.h new file mode 100644 index 00000000000..8f8634bfaa0 --- /dev/null +++ b/third_party/upb/upb/handlers-inl.h @@ -0,0 +1,923 @@ +/* +** Inline definitions for handlers.h, which are particularly long and a bit +** tricky. +*/ + +#ifndef UPB_HANDLERS_INL_H_ +#define UPB_HANDLERS_INL_H_ + +#include +#include +#include "upb/handlers.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus + +/* Type detection and typedefs for integer types. + * For platforms where there are multiple 32-bit or 64-bit types, we need to be + * able to enumerate them so we can properly create overloads for all variants. + * + * If any platform existed where there were three integer types with the same + * size, this would have to become more complicated. For example, short, int, + * and long could all be 32-bits. Even more diabolically, short, int, long, + * and long long could all be 64 bits and still be standard-compliant. + * However, few platforms are this strange, and it's unlikely that upb will be + * used on the strangest ones. */ + +/* Can't count on stdint.h limits like INT32_MAX, because in C++ these are + * only defined when __STDC_LIMIT_MACROS are defined before the *first* include + * of stdint.h. We can't guarantee that someone else didn't include these first + * without defining __STDC_LIMIT_MACROS. */ +#define UPB_INT32_MAX 0x7fffffffLL +#define UPB_INT32_MIN (-UPB_INT32_MAX - 1) +#define UPB_INT64_MAX 0x7fffffffffffffffLL +#define UPB_INT64_MIN (-UPB_INT64_MAX - 1) + +#if INT_MAX == UPB_INT32_MAX && INT_MIN == UPB_INT32_MIN +#define UPB_INT_IS_32BITS 1 +#endif + +#if LONG_MAX == UPB_INT32_MAX && LONG_MIN == UPB_INT32_MIN +#define UPB_LONG_IS_32BITS 1 +#endif + +#if LONG_MAX == UPB_INT64_MAX && LONG_MIN == UPB_INT64_MIN +#define UPB_LONG_IS_64BITS 1 +#endif + +#if LLONG_MAX == UPB_INT64_MAX && LLONG_MIN == UPB_INT64_MIN +#define UPB_LLONG_IS_64BITS 1 +#endif + +/* We use macros instead of typedefs so we can undefine them later and avoid + * leaking them outside this header file. */ +#if UPB_INT_IS_32BITS +#define UPB_INT32_T int +#define UPB_UINT32_T unsigned int + +#if UPB_LONG_IS_32BITS +#define UPB_TWO_32BIT_TYPES 1 +#define UPB_INT32ALT_T long +#define UPB_UINT32ALT_T unsigned long +#endif /* UPB_LONG_IS_32BITS */ + +#elif UPB_LONG_IS_32BITS /* && !UPB_INT_IS_32BITS */ +#define UPB_INT32_T long +#define UPB_UINT32_T unsigned long +#endif /* UPB_INT_IS_32BITS */ + + +#if UPB_LONG_IS_64BITS +#define UPB_INT64_T long +#define UPB_UINT64_T unsigned long + +#if UPB_LLONG_IS_64BITS +#define UPB_TWO_64BIT_TYPES 1 +#define UPB_INT64ALT_T long long +#define UPB_UINT64ALT_T unsigned long long +#endif /* UPB_LLONG_IS_64BITS */ + +#elif UPB_LLONG_IS_64BITS /* && !UPB_LONG_IS_64BITS */ +#define UPB_INT64_T long long +#define UPB_UINT64_T unsigned long long +#endif /* UPB_LONG_IS_64BITS */ + +#undef UPB_INT32_MAX +#undef UPB_INT32_MIN +#undef UPB_INT64_MAX +#undef UPB_INT64_MIN +#undef UPB_INT_IS_32BITS +#undef UPB_LONG_IS_32BITS +#undef UPB_LONG_IS_64BITS +#undef UPB_LLONG_IS_64BITS + + +namespace upb { + +typedef void CleanupFunc(void *ptr); + +/* Template to remove "const" from "const T*" and just return "T*". + * + * We define a nonsense default because otherwise it will fail to instantiate as + * a function parameter type even in cases where we don't expect any caller to + * actually match the overload. */ +class CouldntRemoveConst {}; +template struct remove_constptr { typedef CouldntRemoveConst type; }; +template struct remove_constptr { typedef T *type; }; + +/* Template that we use below to remove a template specialization from + * consideration if it matches a specific type. */ +template struct disable_if_same { typedef void Type; }; +template struct disable_if_same {}; + +template void DeletePointer(void *p) { delete static_cast(p); } + +template +struct FirstUnlessVoidOrBool { + typedef T1 value; +}; + +template +struct FirstUnlessVoidOrBool { + typedef T2 value; +}; + +template +struct FirstUnlessVoidOrBool { + typedef T2 value; +}; + +template +struct is_same { + static bool value; +}; + +template +struct is_same { + static bool value; +}; + +template +bool is_same::value = false; + +template +bool is_same::value = true; + +/* FuncInfo *******************************************************************/ + +/* Info about the user's original, pre-wrapped function. */ +template +struct FuncInfo { + /* The type of the closure that the function takes (its first param). */ + typedef C Closure; + + /* The return type. */ + typedef R Return; +}; + +/* Func ***********************************************************************/ + +/* Func1, Func2, Func3: Template classes representing a function and its + * signature. + * + * Since the function is a template parameter, calling the function can be + * inlined at compile-time and does not require a function pointer at runtime. + * These functions are not bound to a handler data so have no data or cleanup + * handler. */ +struct UnboundFunc { + CleanupFunc *GetCleanup() { return nullptr; } + void *GetData() { return nullptr; } +}; + +template +struct Func1 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1) { return F(p1); } +}; + +template +struct Func2 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2) { return F(p1, p2); } +}; + +template +struct Func3 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2, P3 p3) { return F(p1, p2, p3); } +}; + +template +struct Func4 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2, P3 p3, P4 p4) { return F(p1, p2, p3, p4); } +}; + +template +struct Func5 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) { + return F(p1, p2, p3, p4, p5); + } +}; + +/* BoundFunc ******************************************************************/ + +/* BoundFunc2, BoundFunc3: Like Func2/Func3 except also contains a value that + * shall be bound to the function's second parameter. + * + * Note that the second parameter is a const pointer, but our stored bound value + * is non-const so we can free it when the handlers are destroyed. */ +template +struct BoundFunc { + typedef typename remove_constptr::type MutableP2; + explicit BoundFunc(MutableP2 data_) : data(data_) {} + CleanupFunc *GetCleanup() { return &DeletePointer; } + MutableP2 GetData() { return data; } + MutableP2 data; +}; + +template +struct BoundFunc2 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc2(typename Base::MutableP2 arg) : Base(arg) {} +}; + +template +struct BoundFunc3 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc3(typename Base::MutableP2 arg) : Base(arg) {} +}; + +template +struct BoundFunc4 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc4(typename Base::MutableP2 arg) : Base(arg) {} +}; + +template +struct BoundFunc5 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc5(typename Base::MutableP2 arg) : Base(arg) {} +}; + +/* FuncSig ********************************************************************/ + +/* FuncSig1, FuncSig2, FuncSig3: template classes reflecting a function + * *signature*, but without a specific function attached. + * + * These classes contain member functions that can be invoked with a + * specific function to return a Func/BoundFunc class. */ +template +struct FuncSig1 { + template + Func1 > GetFunc() { + return Func1 >(); + } +}; + +template +struct FuncSig2 { + template + Func2 > GetFunc() { + return Func2 >(); + } + + template + BoundFunc2 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc2 >(param2); + } +}; + +template +struct FuncSig3 { + template + Func3 > GetFunc() { + return Func3 >(); + } + + template + BoundFunc3 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc3 >(param2); + } +}; + +template +struct FuncSig4 { + template + Func4 > GetFunc() { + return Func4 >(); + } + + template + BoundFunc4 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc4 >(param2); + } +}; + +template +struct FuncSig5 { + template + Func5 > GetFunc() { + return Func5 >(); + } + + template + BoundFunc5 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc5 >(param2); + } +}; + +/* Overloaded template function that can construct the appropriate FuncSig* + * class given a function pointer by deducing the template parameters. */ +template +inline FuncSig1 MatchFunc(R (*f)(P1)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig1(); +} + +template +inline FuncSig2 MatchFunc(R (*f)(P1, P2)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig2(); +} + +template +inline FuncSig3 MatchFunc(R (*f)(P1, P2, P3)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig3(); +} + +template +inline FuncSig4 MatchFunc(R (*f)(P1, P2, P3, P4)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig4(); +} + +template +inline FuncSig5 MatchFunc(R (*f)(P1, P2, P3, P4, P5)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig5(); +} + +/* MethodSig ******************************************************************/ + +/* CallMethod*: a function template that calls a given method. */ +template +R CallMethod0(C *obj) { + return ((*obj).*F)(); +} + +template +R CallMethod1(C *obj, P1 arg1) { + return ((*obj).*F)(arg1); +} + +template +R CallMethod2(C *obj, P1 arg1, P2 arg2) { + return ((*obj).*F)(arg1, arg2); +} + +template +R CallMethod3(C *obj, P1 arg1, P2 arg2, P3 arg3) { + return ((*obj).*F)(arg1, arg2, arg3); +} + +template +R CallMethod4(C *obj, P1 arg1, P2 arg2, P3 arg3, P4 arg4) { + return ((*obj).*F)(arg1, arg2, arg3, arg4); +} + +/* MethodSig: like FuncSig, but for member functions. + * + * GetFunc() returns a normal FuncN object, so after calling GetFunc() no + * more logic is required to special-case methods. */ +template +struct MethodSig0 { + template + Func1, FuncInfo > GetFunc() { + return Func1, FuncInfo >(); + } +}; + +template +struct MethodSig1 { + template + Func2, FuncInfo > GetFunc() { + return Func2, FuncInfo >(); + } + + template + BoundFunc2, FuncInfo > GetFunc( + typename remove_constptr::type param1) { + return BoundFunc2, FuncInfo >( + param1); + } +}; + +template +struct MethodSig2 { + template + Func3, FuncInfo > + GetFunc() { + return Func3, + FuncInfo >(); + } + + template + BoundFunc3, FuncInfo > + GetFunc(typename remove_constptr::type param1) { + return BoundFunc3, + FuncInfo >(param1); + } +}; + +template +struct MethodSig3 { + template + Func4, FuncInfo > + GetFunc() { + return Func4, + FuncInfo >(); + } + + template + BoundFunc4, + FuncInfo > + GetFunc(typename remove_constptr::type param1) { + return BoundFunc4, + FuncInfo >(param1); + } +}; + +template +struct MethodSig4 { + template + Func5, + FuncInfo > + GetFunc() { + return Func5, + FuncInfo >(); + } + + template + BoundFunc5, + FuncInfo > + GetFunc(typename remove_constptr::type param1) { + return BoundFunc5, FuncInfo >( + param1); + } +}; + +template +inline MethodSig0 MatchFunc(R (C::*f)()) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig0(); +} + +template +inline MethodSig1 MatchFunc(R (C::*f)(P1)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig1(); +} + +template +inline MethodSig2 MatchFunc(R (C::*f)(P1, P2)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig2(); +} + +template +inline MethodSig3 MatchFunc(R (C::*f)(P1, P2, P3)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig3(); +} + +template +inline MethodSig4 MatchFunc(R (C::*f)(P1, P2, P3, P4)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig4(); +} + +/* MaybeWrapReturn ************************************************************/ + +/* Template class that attempts to wrap the return value of the function so it + * matches the expected type. There are two main adjustments it may make: + * + * 1. If the function returns void, make it return the expected type and with + * a value that always indicates success. + * 2. If the function returns bool, make it return the expected type with a + * value that indicates success or failure. + * + * The "expected type" for return is: + * 1. void* for start handlers. If the closure parameter has a different type + * we will cast it to void* for the return in the success case. + * 2. size_t for string buffer handlers. + * 3. bool for everything else. */ + +/* Template parameters are FuncN type and desired return type. */ +template +struct MaybeWrapReturn; + +/* If the return type matches, return the given function unwrapped. */ +template +struct MaybeWrapReturn { + typedef F Func; +}; + +/* Function wrapper that munges the return value from void to (bool)true. */ +template +bool ReturnTrue2(P1 p1, P2 p2) { + F(p1, p2); + return true; +} + +template +bool ReturnTrue3(P1 p1, P2 p2, P3 p3) { + F(p1, p2, p3); + return true; +} + +/* Function wrapper that munges the return value from void to (void*)arg1 */ +template +void *ReturnClosure2(P1 p1, P2 p2) { + F(p1, p2); + return p1; +} + +template +void *ReturnClosure3(P1 p1, P2 p2, P3 p3) { + F(p1, p2, p3); + return p1; +} + +/* Function wrapper that munges the return value from R to void*. */ +template +void *CastReturnToVoidPtr2(P1 p1, P2 p2) { + return F(p1, p2); +} + +template +void *CastReturnToVoidPtr3(P1 p1, P2 p2, P3 p3) { + return F(p1, p2, p3); +} + +/* Function wrapper that munges the return value from bool to void*. */ +template +void *ReturnClosureOrBreak2(P1 p1, P2 p2) { + return F(p1, p2) ? p1 : UPB_BREAK; +} + +template +void *ReturnClosureOrBreak3(P1 p1, P2 p2, P3 p3) { + return F(p1, p2, p3) ? p1 : UPB_BREAK; +} + +/* For the string callback, which takes five params, returns the size param. */ +template +size_t ReturnStringLen(P1 p1, P2 p2, const char *p3, size_t p4, + const upb_bufhandle *p5) { + F(p1, p2, p3, p4, p5); + return p4; +} + +/* For the string callback, which takes five params, returns the size param or + * zero. */ +template +size_t ReturnNOr0(P1 p1, P2 p2, const char *p3, size_t p4, + const upb_bufhandle *p5) { + return F(p1, p2, p3, p4, p5) ? p4 : 0; +} + +/* If we have a function returning void but want a function returning bool, wrap + * it in a function that returns true. */ +template +struct MaybeWrapReturn, bool> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, bool> { + typedef Func3, I> Func; +}; + +/* If our function returns void but we want one returning void*, wrap it in a + * function that returns the first argument. */ +template +struct MaybeWrapReturn, void *> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, void *> { + typedef Func3, I> Func; +}; + +/* If our function returns R* but we want one returning void*, wrap it in a + * function that casts to void*. */ +template +struct MaybeWrapReturn, void *, + typename disable_if_same::Type> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, void *, + typename disable_if_same::Type> { + typedef Func3, I> + Func; +}; + +/* If our function returns bool but we want one returning void*, wrap it in a + * function that returns either the first param or UPB_BREAK. */ +template +struct MaybeWrapReturn, void *> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, void *> { + typedef Func3, I> + Func; +}; + +/* If our function returns void but we want one returning size_t, wrap it in a + * function that returns the size argument. */ +template +struct MaybeWrapReturn< + Func5, + size_t> { + typedef Func5, I> Func; +}; + +/* If our function returns bool but we want one returning size_t, wrap it in a + * function that returns either 0 or the buf size. */ +template +struct MaybeWrapReturn< + Func5, + size_t> { + typedef Func5, I> Func; +}; + +/* ConvertParams **************************************************************/ + +/* Template class that converts the function parameters if necessary, and + * ignores the HandlerData parameter if appropriate. + * + * Template parameter is the are FuncN function type. */ +template +struct ConvertParams; + +/* Function that discards the handler data parameter. */ +template +R IgnoreHandlerData2(void *p1, const void *hd) { + UPB_UNUSED(hd); + return F(static_cast(p1)); +} + +template +R IgnoreHandlerData3(void *p1, const void *hd, P2Wrapper p2) { + UPB_UNUSED(hd); + return F(static_cast(p1), p2); +} + +template +R IgnoreHandlerData4(void *p1, const void *hd, P2 p2, P3 p3) { + UPB_UNUSED(hd); + return F(static_cast(p1), p2, p3); +} + +template +R IgnoreHandlerData5(void *p1, const void *hd, P2 p2, P3 p3, P4 p4) { + UPB_UNUSED(hd); + return F(static_cast(p1), p2, p3, p4); +} + +template +R IgnoreHandlerDataIgnoreHandle(void *p1, const void *hd, const char *p2, + size_t p3, const upb_bufhandle *handle) { + UPB_UNUSED(hd); + UPB_UNUSED(handle); + return F(static_cast(p1), p2, p3); +} + +/* Function that casts the handler data parameter. */ +template +R CastHandlerData2(void *c, const void *hd) { + return F(static_cast(c), static_cast(hd)); +} + +template +R CastHandlerData3(void *c, const void *hd, P3Wrapper p3) { + return F(static_cast(c), static_cast(hd), p3); +} + +template +R CastHandlerData5(void *c, const void *hd, P3 p3, P4 p4, P5 p5) { + return F(static_cast(c), static_cast(hd), p3, p4, p5); +} + +template +R CastHandlerDataIgnoreHandle(void *c, const void *hd, const char *p3, + size_t p4, const upb_bufhandle *handle) { + UPB_UNUSED(handle); + return F(static_cast(c), static_cast(hd), p3, p4); +} + +/* For unbound functions, ignore the handler data. */ +template +struct ConvertParams, T> { + typedef Func2, I> Func; +}; + +template +struct ConvertParams, + R2 (*)(P1_2, P2_2, P3_2)> { + typedef Func3, I> Func; +}; + +/* For StringBuffer only; this ignores both the handler data and the + * upb_bufhandle. */ +template +struct ConvertParams, T> { + typedef Func5, + I> Func; +}; + +template +struct ConvertParams, T> { + typedef Func5, I> Func; +}; + +/* For bound functions, cast the handler data. */ +template +struct ConvertParams, T> { + typedef Func2, I> + Func; +}; + +template +struct ConvertParams, + R2 (*)(P1_2, P2_2, P3_2)> { + typedef Func3, I> Func; +}; + +/* For StringBuffer only; this ignores the upb_bufhandle. */ +template +struct ConvertParams, T> { + typedef Func5, I> + Func; +}; + +template +struct ConvertParams, T> { + typedef Func5, I> Func; +}; + +/* utype/ltype are upper/lower-case, ctype is canonical C type, vtype is + * variant C type. */ +#define TYPE_METHODS(utype, ltype, ctype, vtype) \ + template <> \ + struct CanonicalType { \ + typedef ctype Type; \ + }; \ + template <> \ + inline bool HandlersPtr::SetValueHandler( \ + FieldDefPtr f, const HandlersPtr::utype##Handler &handler) { \ + handler.AddCleanup(ptr()); \ + return upb_handlers_set##ltype(ptr(), f.ptr(), handler.handler(), \ + &handler.attr()); \ + } + +TYPE_METHODS(Double, double, double, double) +TYPE_METHODS(Float, float, float, float) +TYPE_METHODS(UInt64, uint64, uint64_t, UPB_UINT64_T) +TYPE_METHODS(UInt32, uint32, uint32_t, UPB_UINT32_T) +TYPE_METHODS(Int64, int64, int64_t, UPB_INT64_T) +TYPE_METHODS(Int32, int32, int32_t, UPB_INT32_T) +TYPE_METHODS(Bool, bool, bool, bool) + +#ifdef UPB_TWO_32BIT_TYPES +TYPE_METHODS(Int32, int32, int32_t, UPB_INT32ALT_T) +TYPE_METHODS(UInt32, uint32, uint32_t, UPB_UINT32ALT_T) +#endif + +#ifdef UPB_TWO_64BIT_TYPES +TYPE_METHODS(Int64, int64, int64_t, UPB_INT64ALT_T) +TYPE_METHODS(UInt64, uint64, uint64_t, UPB_UINT64ALT_T) +#endif +#undef TYPE_METHODS + +template <> struct CanonicalType { + typedef Status* Type; +}; + +template struct ReturnOf; + +template +struct ReturnOf { + typedef R Return; +}; + +template +struct ReturnOf { + typedef R Return; +}; + +template +struct ReturnOf { + typedef R Return; +}; + +template +struct ReturnOf { + typedef R Return; +}; + + +template +template +inline Handler::Handler(F func) + : registered_(false), + cleanup_data_(func.GetData()), + cleanup_func_(func.GetCleanup()) { + attr_.handler_data = func.GetData(); + typedef typename ReturnOf::Return Return; + typedef typename ConvertParams::Func ConvertedParamsFunc; + typedef typename MaybeWrapReturn::Func + ReturnWrappedFunc; + handler_ = ReturnWrappedFunc().Call; + + /* Set attributes based on what templates can statically tell us about the + * user's function. */ + + /* If the original function returns void, then we know that we wrapped it to + * always return ok. */ + bool always_ok = is_same::value; + attr_.alwaysok = always_ok; + + /* Closure parameter and return type. */ + attr_.closure_type = UniquePtrForType(); + + /* We use the closure type (from the first parameter) if the return type is + * void or bool, since these are the two cases we wrap to return the closure's + * type anyway. + * + * This is all nonsense for non START* handlers, but it doesn't matter because + * in that case the value will be ignored. */ + typedef typename FirstUnlessVoidOrBool::value + EffectiveReturn; + attr_.return_closure_type = UniquePtrForType(); +} + +template +inline void Handler::AddCleanup(upb_handlers* h) const { + UPB_ASSERT(!registered_); + registered_ = true; + if (cleanup_func_) { + bool ok = upb_handlers_addcleanup(h, cleanup_data_, cleanup_func_); + UPB_ASSERT(ok); + } +} + +} /* namespace upb */ + +#endif /* __cplusplus */ + + +#undef UPB_TWO_32BIT_TYPES +#undef UPB_TWO_64BIT_TYPES +#undef UPB_INT32_T +#undef UPB_UINT32_T +#undef UPB_INT32ALT_T +#undef UPB_UINT32ALT_T +#undef UPB_INT64_T +#undef UPB_UINT64_T +#undef UPB_INT64ALT_T +#undef UPB_UINT64ALT_T + +#include "upb/port_undef.inc" + +#endif /* UPB_HANDLERS_INL_H_ */ diff --git a/third_party/upb/upb/handlers.c b/third_party/upb/upb/handlers.c new file mode 100644 index 00000000000..844a3601765 --- /dev/null +++ b/third_party/upb/upb/handlers.c @@ -0,0 +1,567 @@ +/* +** TODO(haberman): it's unclear whether a lot of the consistency checks should +** UPB_ASSERT() or return false. +*/ + +#include "upb/handlers.h" + +#include + +#include "upb/sink.h" + +#include "upb/port_def.inc" + +struct upb_handlers { + upb_handlercache *cache; + const upb_msgdef *msg; + const upb_handlers **sub; + const void *top_closure_type; + upb_handlers_tabent table[1]; /* Dynamically-sized field handler array. */ +}; + +static void *upb_calloc(upb_arena *arena, size_t size) { + void *mem = upb_malloc(upb_arena_alloc(arena), size); + if (mem) { + memset(mem, 0, size); + } + return mem; +} + +/* Defined for the sole purpose of having a unique pointer value for + * UPB_NO_CLOSURE. */ +char _upb_noclosure; + +/* Given a selector for a STARTSUBMSG handler, resolves to a pointer to the + * subhandlers for this submessage field. */ +#define SUBH(h, selector) (h->sub[selector]) + +/* The selector for a submessage field is the field index. */ +#define SUBH_F(h, f) SUBH(h, upb_fielddef_index(f)) + +static int32_t trygetsel(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + upb_selector_t sel; + bool ok; + + ok = upb_handlers_getselector(f, type, &sel); + + UPB_ASSERT(upb_handlers_msgdef(h) == upb_fielddef_containingtype(f)); + UPB_ASSERT(ok); + + return sel; +} + +static upb_selector_t handlers_getsel(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + int32_t sel = trygetsel(h, f, type); + UPB_ASSERT(sel >= 0); + return sel; +} + +static const void **returntype(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + return &h->table[handlers_getsel(h, f, type)].attr.return_closure_type; +} + +static bool doset(upb_handlers *h, int32_t sel, const upb_fielddef *f, + upb_handlertype_t type, upb_func *func, + const upb_handlerattr *attr) { + upb_handlerattr set_attr = UPB_HANDLERATTR_INIT; + const void *closure_type; + const void **context_closure_type; + + UPB_ASSERT(!h->table[sel].func); + + if (attr) { + set_attr = *attr; + } + + /* Check that the given closure type matches the closure type that has been + * established for this context (if any). */ + closure_type = set_attr.closure_type; + + if (type == UPB_HANDLER_STRING) { + context_closure_type = returntype(h, f, UPB_HANDLER_STARTSTR); + } else if (f && upb_fielddef_isseq(f) && + type != UPB_HANDLER_STARTSEQ && + type != UPB_HANDLER_ENDSEQ) { + context_closure_type = returntype(h, f, UPB_HANDLER_STARTSEQ); + } else { + context_closure_type = &h->top_closure_type; + } + + if (closure_type && *context_closure_type && + closure_type != *context_closure_type) { + return false; + } + + if (closure_type) + *context_closure_type = closure_type; + + /* If this is a STARTSEQ or STARTSTR handler, check that the returned pointer + * matches any pre-existing expectations about what type is expected. */ + if (type == UPB_HANDLER_STARTSEQ || type == UPB_HANDLER_STARTSTR) { + const void *return_type = set_attr.return_closure_type; + const void *table_return_type = h->table[sel].attr.return_closure_type; + if (return_type && table_return_type && return_type != table_return_type) { + return false; + } + + if (table_return_type && !return_type) { + set_attr.return_closure_type = table_return_type; + } + } + + h->table[sel].func = (upb_func*)func; + h->table[sel].attr = set_attr; + return true; +} + +/* Returns the effective closure type for this handler (which will propagate + * from outer frames if this frame has no START* handler). Not implemented for + * UPB_HANDLER_STRING at the moment since this is not needed. Returns NULL is + * the effective closure type is unspecified (either no handler was registered + * to specify it or the handler that was registered did not specify the closure + * type). */ +const void *effective_closure_type(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + const void *ret; + upb_selector_t sel; + + UPB_ASSERT(type != UPB_HANDLER_STRING); + ret = h->top_closure_type; + + if (upb_fielddef_isseq(f) && + type != UPB_HANDLER_STARTSEQ && + type != UPB_HANDLER_ENDSEQ && + h->table[sel = handlers_getsel(h, f, UPB_HANDLER_STARTSEQ)].func) { + ret = h->table[sel].attr.return_closure_type; + } + + if (type == UPB_HANDLER_STRING && + h->table[sel = handlers_getsel(h, f, UPB_HANDLER_STARTSTR)].func) { + ret = h->table[sel].attr.return_closure_type; + } + + /* The effective type of the submessage; not used yet. + * if (type == SUBMESSAGE && + * h->table[sel = handlers_getsel(h, f, UPB_HANDLER_STARTSUBMSG)].func) { + * ret = h->table[sel].attr.return_closure_type; + * } */ + + return ret; +} + +/* Checks whether the START* handler specified by f & type is missing even + * though it is required to convert the established type of an outer frame + * ("closure_type") into the established type of an inner frame (represented in + * the return closure type of this handler's attr. */ +bool checkstart(upb_handlers *h, const upb_fielddef *f, upb_handlertype_t type, + upb_status *status) { + const void *closure_type; + const upb_handlerattr *attr; + const void *return_closure_type; + + upb_selector_t sel = handlers_getsel(h, f, type); + if (h->table[sel].func) return true; + closure_type = effective_closure_type(h, f, type); + attr = &h->table[sel].attr; + return_closure_type = attr->return_closure_type; + if (closure_type && return_closure_type && + closure_type != return_closure_type) { + return false; + } + return true; +} + +static upb_handlers *upb_handlers_new(const upb_msgdef *md, + upb_handlercache *cache, + upb_arena *arena) { + int extra; + upb_handlers *h; + + extra = sizeof(upb_handlers_tabent) * (upb_msgdef_selectorcount(md) - 1); + h = upb_calloc(arena, sizeof(*h) + extra); + if (!h) return NULL; + + h->cache = cache; + h->msg = md; + + if (upb_msgdef_submsgfieldcount(md) > 0) { + size_t bytes = upb_msgdef_submsgfieldcount(md) * sizeof(*h->sub); + h->sub = upb_calloc(arena, bytes); + if (!h->sub) return NULL; + } else { + h->sub = 0; + } + + /* calloc() above initialized all handlers to NULL. */ + return h; +} + +/* Public interface ***********************************************************/ + +#define SETTER(name, handlerctype, handlertype) \ + bool upb_handlers_set##name(upb_handlers *h, const upb_fielddef *f, \ + handlerctype func, \ + const upb_handlerattr *attr) { \ + int32_t sel = trygetsel(h, f, handlertype); \ + return doset(h, sel, f, handlertype, (upb_func *)func, attr); \ + } + +SETTER(int32, upb_int32_handlerfunc*, UPB_HANDLER_INT32) +SETTER(int64, upb_int64_handlerfunc*, UPB_HANDLER_INT64) +SETTER(uint32, upb_uint32_handlerfunc*, UPB_HANDLER_UINT32) +SETTER(uint64, upb_uint64_handlerfunc*, UPB_HANDLER_UINT64) +SETTER(float, upb_float_handlerfunc*, UPB_HANDLER_FLOAT) +SETTER(double, upb_double_handlerfunc*, UPB_HANDLER_DOUBLE) +SETTER(bool, upb_bool_handlerfunc*, UPB_HANDLER_BOOL) +SETTER(startstr, upb_startstr_handlerfunc*, UPB_HANDLER_STARTSTR) +SETTER(string, upb_string_handlerfunc*, UPB_HANDLER_STRING) +SETTER(endstr, upb_endfield_handlerfunc*, UPB_HANDLER_ENDSTR) +SETTER(startseq, upb_startfield_handlerfunc*, UPB_HANDLER_STARTSEQ) +SETTER(startsubmsg, upb_startfield_handlerfunc*, UPB_HANDLER_STARTSUBMSG) +SETTER(endsubmsg, upb_endfield_handlerfunc*, UPB_HANDLER_ENDSUBMSG) +SETTER(endseq, upb_endfield_handlerfunc*, UPB_HANDLER_ENDSEQ) + +#undef SETTER + +bool upb_handlers_setunknown(upb_handlers *h, upb_unknown_handlerfunc *func, + const upb_handlerattr *attr) { + return doset(h, UPB_UNKNOWN_SELECTOR, NULL, UPB_HANDLER_INT32, + (upb_func *)func, attr); +} + +bool upb_handlers_setstartmsg(upb_handlers *h, upb_startmsg_handlerfunc *func, + const upb_handlerattr *attr) { + return doset(h, UPB_STARTMSG_SELECTOR, NULL, UPB_HANDLER_INT32, + (upb_func *)func, attr); +} + +bool upb_handlers_setendmsg(upb_handlers *h, upb_endmsg_handlerfunc *func, + const upb_handlerattr *attr) { + return doset(h, UPB_ENDMSG_SELECTOR, NULL, UPB_HANDLER_INT32, + (upb_func *)func, attr); +} + +bool upb_handlers_setsubhandlers(upb_handlers *h, const upb_fielddef *f, + const upb_handlers *sub) { + UPB_ASSERT(sub); + UPB_ASSERT(upb_fielddef_issubmsg(f)); + if (SUBH_F(h, f)) return false; /* Can't reset. */ + if (upb_handlers_msgdef(sub) != upb_fielddef_msgsubdef(f)) { + return false; + } + SUBH_F(h, f) = sub; + return true; +} + +const upb_handlers *upb_handlers_getsubhandlers(const upb_handlers *h, + const upb_fielddef *f) { + UPB_ASSERT(upb_fielddef_issubmsg(f)); + return SUBH_F(h, f); +} + +upb_func *upb_handlers_gethandler(const upb_handlers *h, upb_selector_t s, + const void **handler_data) { + upb_func *ret = (upb_func *)h->table[s].func; + if (ret && handler_data) { + *handler_data = h->table[s].attr.handler_data; + } + return ret; +} + +bool upb_handlers_getattr(const upb_handlers *h, upb_selector_t sel, + upb_handlerattr *attr) { + if (!upb_handlers_gethandler(h, sel, NULL)) + return false; + *attr = h->table[sel].attr; + return true; +} + +const upb_handlers *upb_handlers_getsubhandlers_sel(const upb_handlers *h, + upb_selector_t sel) { + /* STARTSUBMSG selector in sel is the field's selector base. */ + return SUBH(h, sel - UPB_STATIC_SELECTOR_COUNT); +} + +const upb_msgdef *upb_handlers_msgdef(const upb_handlers *h) { return h->msg; } + +bool upb_handlers_addcleanup(upb_handlers *h, void *p, upb_handlerfree *func) { + return upb_handlercache_addcleanup(h->cache, p, func); +} + +upb_handlertype_t upb_handlers_getprimitivehandlertype(const upb_fielddef *f) { + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: return UPB_HANDLER_INT32; + case UPB_TYPE_INT64: return UPB_HANDLER_INT64; + case UPB_TYPE_UINT32: return UPB_HANDLER_UINT32; + case UPB_TYPE_UINT64: return UPB_HANDLER_UINT64; + case UPB_TYPE_FLOAT: return UPB_HANDLER_FLOAT; + case UPB_TYPE_DOUBLE: return UPB_HANDLER_DOUBLE; + case UPB_TYPE_BOOL: return UPB_HANDLER_BOOL; + default: UPB_ASSERT(false); return -1; /* Invalid input. */ + } +} + +bool upb_handlers_getselector(const upb_fielddef *f, upb_handlertype_t type, + upb_selector_t *s) { + uint32_t selector_base = upb_fielddef_selectorbase(f); + switch (type) { + case UPB_HANDLER_INT32: + case UPB_HANDLER_INT64: + case UPB_HANDLER_UINT32: + case UPB_HANDLER_UINT64: + case UPB_HANDLER_FLOAT: + case UPB_HANDLER_DOUBLE: + case UPB_HANDLER_BOOL: + if (!upb_fielddef_isprimitive(f) || + upb_handlers_getprimitivehandlertype(f) != type) + return false; + *s = selector_base; + break; + case UPB_HANDLER_STRING: + if (upb_fielddef_isstring(f)) { + *s = selector_base; + } else if (upb_fielddef_lazy(f)) { + *s = selector_base + 3; + } else { + return false; + } + break; + case UPB_HANDLER_STARTSTR: + if (upb_fielddef_isstring(f) || upb_fielddef_lazy(f)) { + *s = selector_base + 1; + } else { + return false; + } + break; + case UPB_HANDLER_ENDSTR: + if (upb_fielddef_isstring(f) || upb_fielddef_lazy(f)) { + *s = selector_base + 2; + } else { + return false; + } + break; + case UPB_HANDLER_STARTSEQ: + if (!upb_fielddef_isseq(f)) return false; + *s = selector_base - 2; + break; + case UPB_HANDLER_ENDSEQ: + if (!upb_fielddef_isseq(f)) return false; + *s = selector_base - 1; + break; + case UPB_HANDLER_STARTSUBMSG: + if (!upb_fielddef_issubmsg(f)) return false; + /* Selectors for STARTSUBMSG are at the beginning of the table so that the + * selector can also be used as an index into the "sub" array of + * subhandlers. The indexes for the two into these two tables are the + * same, except that in the handler table the static selectors come first. */ + *s = upb_fielddef_index(f) + UPB_STATIC_SELECTOR_COUNT; + break; + case UPB_HANDLER_ENDSUBMSG: + if (!upb_fielddef_issubmsg(f)) return false; + *s = selector_base; + break; + } + UPB_ASSERT((size_t)*s < upb_msgdef_selectorcount(upb_fielddef_containingtype(f))); + return true; +} + +/* upb_handlercache ***********************************************************/ + +struct upb_handlercache { + upb_arena *arena; + upb_inttable tab; /* maps upb_msgdef* -> upb_handlers*. */ + upb_handlers_callback *callback; + const void *closure; +}; + +const upb_handlers *upb_handlercache_get(upb_handlercache *c, + const upb_msgdef *md) { + upb_msg_field_iter i; + upb_value v; + upb_handlers *h; + + if (upb_inttable_lookupptr(&c->tab, md, &v)) { + return upb_value_getptr(v); + } + + h = upb_handlers_new(md, c, c->arena); + v = upb_value_ptr(h); + + if (!h) return NULL; + if (!upb_inttable_insertptr(&c->tab, md, v)) return NULL; + + c->callback(c->closure, h); + + /* For each submessage field, get or create a handlers object and set it as + * the subhandlers. */ + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + + if (upb_fielddef_issubmsg(f)) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + const upb_handlers *sub_mh = upb_handlercache_get(c, subdef); + + if (!sub_mh) return NULL; + + upb_handlers_setsubhandlers(h, f, sub_mh); + } + } + + return h; +} + + +upb_handlercache *upb_handlercache_new(upb_handlers_callback *callback, + const void *closure) { + upb_handlercache *cache = upb_gmalloc(sizeof(*cache)); + + if (!cache) return NULL; + + cache->arena = upb_arena_new(); + + cache->callback = callback; + cache->closure = closure; + + if (!upb_inttable_init(&cache->tab, UPB_CTYPE_PTR)) goto oom; + + return cache; + +oom: + upb_gfree(cache); + return NULL; +} + +void upb_handlercache_free(upb_handlercache *cache) { + upb_inttable_uninit(&cache->tab); + upb_arena_free(cache->arena); + upb_gfree(cache); +} + +bool upb_handlercache_addcleanup(upb_handlercache *c, void *p, + upb_handlerfree *func) { + return upb_arena_addcleanup(c->arena, p, func); +} + +/* upb_byteshandler ***********************************************************/ + +bool upb_byteshandler_setstartstr(upb_byteshandler *h, + upb_startstr_handlerfunc *func, void *d) { + h->table[UPB_STARTSTR_SELECTOR].func = (upb_func*)func; + h->table[UPB_STARTSTR_SELECTOR].attr.handler_data = d; + return true; +} + +bool upb_byteshandler_setstring(upb_byteshandler *h, + upb_string_handlerfunc *func, void *d) { + h->table[UPB_STRING_SELECTOR].func = (upb_func*)func; + h->table[UPB_STRING_SELECTOR].attr.handler_data = d; + return true; +} + +bool upb_byteshandler_setendstr(upb_byteshandler *h, + upb_endfield_handlerfunc *func, void *d) { + h->table[UPB_ENDSTR_SELECTOR].func = (upb_func*)func; + h->table[UPB_ENDSTR_SELECTOR].attr.handler_data = d; + return true; +} + +/** Handlers for upb_msg ******************************************************/ + +typedef struct { + size_t offset; + int32_t hasbit; +} upb_msg_handlerdata; + +/* Fallback implementation if the handler is not specialized by the producer. */ +#define MSG_WRITER(type, ctype) \ + bool upb_msg_set ## type (void *c, const void *hd, ctype val) { \ + uint8_t *m = c; \ + const upb_msg_handlerdata *d = hd; \ + if (d->hasbit > 0) \ + *(uint8_t*)&m[d->hasbit / 8] |= 1 << (d->hasbit % 8); \ + *(ctype*)&m[d->offset] = val; \ + return true; \ + } \ + +MSG_WRITER(double, double) +MSG_WRITER(float, float) +MSG_WRITER(int32, int32_t) +MSG_WRITER(int64, int64_t) +MSG_WRITER(uint32, uint32_t) +MSG_WRITER(uint64, uint64_t) +MSG_WRITER(bool, bool) + +bool upb_msg_setscalarhandler(upb_handlers *h, const upb_fielddef *f, + size_t offset, int32_t hasbit) { + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + bool ok; + + upb_msg_handlerdata *d = upb_gmalloc(sizeof(*d)); + if (!d) return false; + d->offset = offset; + d->hasbit = hasbit; + + attr.handler_data = d; + attr.alwaysok = true; + upb_handlers_addcleanup(h, d, upb_gfree); + +#define TYPE(u, l) \ + case UPB_TYPE_##u: \ + ok = upb_handlers_set##l(h, f, upb_msg_set##l, &attr); break; + + ok = false; + + switch (upb_fielddef_type(f)) { + TYPE(INT64, int64); + TYPE(INT32, int32); + TYPE(ENUM, int32); + TYPE(UINT64, uint64); + TYPE(UINT32, uint32); + TYPE(DOUBLE, double); + TYPE(FLOAT, float); + TYPE(BOOL, bool); + default: UPB_ASSERT(false); break; + } +#undef TYPE + + return ok; +} + +bool upb_msg_getscalarhandlerdata(const upb_handlers *h, + upb_selector_t s, + upb_fieldtype_t *type, + size_t *offset, + int32_t *hasbit) { + const upb_msg_handlerdata *d; + const void *p; + upb_func *f = upb_handlers_gethandler(h, s, &p); + + if ((upb_int64_handlerfunc*)f == upb_msg_setint64) { + *type = UPB_TYPE_INT64; + } else if ((upb_int32_handlerfunc*)f == upb_msg_setint32) { + *type = UPB_TYPE_INT32; + } else if ((upb_uint64_handlerfunc*)f == upb_msg_setuint64) { + *type = UPB_TYPE_UINT64; + } else if ((upb_uint32_handlerfunc*)f == upb_msg_setuint32) { + *type = UPB_TYPE_UINT32; + } else if ((upb_double_handlerfunc*)f == upb_msg_setdouble) { + *type = UPB_TYPE_DOUBLE; + } else if ((upb_float_handlerfunc*)f == upb_msg_setfloat) { + *type = UPB_TYPE_FLOAT; + } else if ((upb_bool_handlerfunc*)f == upb_msg_setbool) { + *type = UPB_TYPE_BOOL; + } else { + return false; + } + + d = p; + *offset = d->offset; + *hasbit = d->hasbit; + return true; +} diff --git a/third_party/upb/upb/handlers.h b/third_party/upb/upb/handlers.h new file mode 100644 index 00000000000..2d2380b634b --- /dev/null +++ b/third_party/upb/upb/handlers.h @@ -0,0 +1,732 @@ +/* +** upb::Handlers (upb_handlers) +** +** A upb_handlers is like a virtual table for a upb_msgdef. Each field of the +** message can have associated functions that will be called when we are +** parsing or visiting a stream of data. This is similar to how handlers work +** in SAX (the Simple API for XML). +** +** The handlers have no idea where the data is coming from, so a single set of +** handlers could be used with two completely different data sources (for +** example, a parser and a visitor over in-memory objects). This decoupling is +** the most important feature of upb, because it allows parsers and serializers +** to be highly reusable. +** +** This is a mixed C/C++ interface that offers a full API to both languages. +** See the top-level README for more information. +*/ + +#ifndef UPB_HANDLERS_H +#define UPB_HANDLERS_H + +#include "upb/def.h" +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +namespace upb { +class HandlersPtr; +class HandlerCache; +template class Handler; +template struct CanonicalType; +} /* namespace upb */ +#endif + + +/* The maximum depth that the handler graph can have. This is a resource limit + * for the C stack since we sometimes need to recursively traverse the graph. + * Cycles are ok; the traversal will stop when it detects a cycle, but we must + * hit the cycle before the maximum depth is reached. + * + * If having a single static limit is too inflexible, we can add another variant + * of Handlers::Freeze that allows specifying this as a parameter. */ +#define UPB_MAX_HANDLER_DEPTH 64 + +/* All the different types of handlers that can be registered. + * Only needed for the advanced functions in upb::Handlers. */ +typedef enum { + UPB_HANDLER_INT32, + UPB_HANDLER_INT64, + UPB_HANDLER_UINT32, + UPB_HANDLER_UINT64, + UPB_HANDLER_FLOAT, + UPB_HANDLER_DOUBLE, + UPB_HANDLER_BOOL, + UPB_HANDLER_STARTSTR, + UPB_HANDLER_STRING, + UPB_HANDLER_ENDSTR, + UPB_HANDLER_STARTSUBMSG, + UPB_HANDLER_ENDSUBMSG, + UPB_HANDLER_STARTSEQ, + UPB_HANDLER_ENDSEQ +} upb_handlertype_t; + +#define UPB_HANDLER_MAX (UPB_HANDLER_ENDSEQ+1) + +#define UPB_BREAK NULL + +/* A convenient definition for when no closure is needed. */ +extern char _upb_noclosure; +#define UPB_NO_CLOSURE &_upb_noclosure + +/* A selector refers to a specific field handler in the Handlers object + * (for example: the STARTSUBMSG handler for field "field15"). */ +typedef int32_t upb_selector_t; + +/* Static selectors for upb::Handlers. */ +#define UPB_STARTMSG_SELECTOR 0 +#define UPB_ENDMSG_SELECTOR 1 +#define UPB_UNKNOWN_SELECTOR 2 +#define UPB_STATIC_SELECTOR_COUNT 3 /* Warning: also in upb/def.c. */ + +/* Static selectors for upb::BytesHandler. */ +#define UPB_STARTSTR_SELECTOR 0 +#define UPB_STRING_SELECTOR 1 +#define UPB_ENDSTR_SELECTOR 2 + +#ifdef __cplusplus +template const void *UniquePtrForType() { + static const char ch = 0; + return &ch; +} +#endif + +/* upb_handlers ************************************************************/ + +/* Handler attributes, to be registered with the handler itself. */ +typedef struct { + const void *handler_data; + const void *closure_type; + const void *return_closure_type; + bool alwaysok; +} upb_handlerattr; + +#define UPB_HANDLERATTR_INIT {NULL, NULL, NULL, false} + +/* Bufhandle, data passed along with a buffer to indicate its provenance. */ +typedef struct { + /* The beginning of the buffer. This may be different than the pointer + * passed to a StringBuf handler because the handler may receive data + * that is from the middle or end of a larger buffer. */ + const char *buf; + + /* The offset within the attached object where this buffer begins. Only + * meaningful if there is an attached object. */ + size_t objofs; + + /* The attached object (if any) and a pointer representing its type. */ + const void *obj; + const void *objtype; + +#ifdef __cplusplus + template + void SetAttachedObject(const T* _obj) { + obj = _obj; + objtype = UniquePtrForType(); + } + + template + const T *GetAttachedObject() const { + return objtype == UniquePtrForType() ? static_cast(obj) + : NULL; + } +#endif +} upb_bufhandle; + +#define UPB_BUFHANDLE_INIT {NULL, 0, NULL, NULL} + +/* Handler function typedefs. */ +typedef void upb_handlerfree(void *d); +typedef bool upb_unknown_handlerfunc(void *c, const void *hd, const char *buf, + size_t n); +typedef bool upb_startmsg_handlerfunc(void *c, const void*); +typedef bool upb_endmsg_handlerfunc(void *c, const void *, upb_status *status); +typedef void* upb_startfield_handlerfunc(void *c, const void *hd); +typedef bool upb_endfield_handlerfunc(void *c, const void *hd); +typedef bool upb_int32_handlerfunc(void *c, const void *hd, int32_t val); +typedef bool upb_int64_handlerfunc(void *c, const void *hd, int64_t val); +typedef bool upb_uint32_handlerfunc(void *c, const void *hd, uint32_t val); +typedef bool upb_uint64_handlerfunc(void *c, const void *hd, uint64_t val); +typedef bool upb_float_handlerfunc(void *c, const void *hd, float val); +typedef bool upb_double_handlerfunc(void *c, const void *hd, double val); +typedef bool upb_bool_handlerfunc(void *c, const void *hd, bool val); +typedef void *upb_startstr_handlerfunc(void *c, const void *hd, + size_t size_hint); +typedef size_t upb_string_handlerfunc(void *c, const void *hd, const char *buf, + size_t n, const upb_bufhandle* handle); + +struct upb_handlers; +typedef struct upb_handlers upb_handlers; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Mutating accessors. */ +const upb_status *upb_handlers_status(upb_handlers *h); +void upb_handlers_clearerr(upb_handlers *h); +const upb_msgdef *upb_handlers_msgdef(const upb_handlers *h); +bool upb_handlers_addcleanup(upb_handlers *h, void *p, upb_handlerfree *hfree); +bool upb_handlers_setunknown(upb_handlers *h, upb_unknown_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartmsg(upb_handlers *h, upb_startmsg_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendmsg(upb_handlers *h, upb_endmsg_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setint32(upb_handlers *h, const upb_fielddef *f, + upb_int32_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setint64(upb_handlers *h, const upb_fielddef *f, + upb_int64_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setuint32(upb_handlers *h, const upb_fielddef *f, + upb_uint32_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setuint64(upb_handlers *h, const upb_fielddef *f, + upb_uint64_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setfloat(upb_handlers *h, const upb_fielddef *f, + upb_float_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setdouble(upb_handlers *h, const upb_fielddef *f, + upb_double_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setbool(upb_handlers *h, const upb_fielddef *f, + upb_bool_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartstr(upb_handlers *h, const upb_fielddef *f, + upb_startstr_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstring(upb_handlers *h, const upb_fielddef *f, + upb_string_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendstr(upb_handlers *h, const upb_fielddef *f, + upb_endfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartseq(upb_handlers *h, const upb_fielddef *f, + upb_startfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartsubmsg(upb_handlers *h, const upb_fielddef *f, + upb_startfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendsubmsg(upb_handlers *h, const upb_fielddef *f, + upb_endfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendseq(upb_handlers *h, const upb_fielddef *f, + upb_endfield_handlerfunc *func, + const upb_handlerattr *attr); + +/* Read-only accessors. */ +const upb_handlers *upb_handlers_getsubhandlers(const upb_handlers *h, + const upb_fielddef *f); +const upb_handlers *upb_handlers_getsubhandlers_sel(const upb_handlers *h, + upb_selector_t sel); +upb_func *upb_handlers_gethandler(const upb_handlers *h, upb_selector_t s, + const void **handler_data); +bool upb_handlers_getattr(const upb_handlers *h, upb_selector_t s, + upb_handlerattr *attr); + +/* "Static" methods */ +upb_handlertype_t upb_handlers_getprimitivehandlertype(const upb_fielddef *f); +bool upb_handlers_getselector(const upb_fielddef *f, upb_handlertype_t type, + upb_selector_t *s); +UPB_INLINE upb_selector_t upb_handlers_getendselector(upb_selector_t start) { + return start + 1; +} + +#ifdef __cplusplus +} /* extern "C" */ + +namespace upb { +typedef upb_handlers Handlers; +} + +/* Convenience macros for creating a Handler object that is wrapped with a + * type-safe wrapper function that converts the "void*" parameters/returns + * of the underlying C API into nice C++ function. + * + * Sample usage: + * void OnValue1(MyClosure* c, const MyHandlerData* d, int32_t val) { + * // do stuff ... + * } + * + * // Handler that doesn't need any data bound to it. + * void OnValue2(MyClosure* c, int32_t val) { + * // do stuff ... + * } + * + * // Handler that returns bool so it can return failure if necessary. + * bool OnValue3(MyClosure* c, int32_t val) { + * // do stuff ... + * return ok; + * } + * + * // Member function handler. + * class MyClosure { + * public: + * void OnValue(int32_t val) { + * // do stuff ... + * } + * }; + * + * // Takes ownership of the MyHandlerData. + * handlers->SetInt32Handler(f1, UpbBind(OnValue1, new MyHandlerData(...))); + * handlers->SetInt32Handler(f2, UpbMakeHandler(OnValue2)); + * handlers->SetInt32Handler(f1, UpbMakeHandler(OnValue3)); + * handlers->SetInt32Handler(f2, UpbMakeHandler(&MyClosure::OnValue)); + */ + +/* In C++11, the "template" disambiguator can appear even outside templates, + * so all calls can safely use this pair of macros. */ + +#define UpbMakeHandler(f) upb::MatchFunc(f).template GetFunc() + +/* We have to be careful to only evaluate "d" once. */ +#define UpbBind(f, d) upb::MatchFunc(f).template GetFunc((d)) + +/* Handler: a struct that contains the (handler, data, deleter) tuple that is + * used to register all handlers. Users can Make() these directly but it's + * more convenient to use the UpbMakeHandler/UpbBind macros above. */ +template class upb::Handler { + public: + /* The underlying, handler function signature that upb uses internally. */ + typedef T FuncPtr; + + /* Intentionally implicit. */ + template Handler(F func); + ~Handler() { UPB_ASSERT(registered_); } + + void AddCleanup(upb_handlers* h) const; + FuncPtr handler() const { return handler_; } + const upb_handlerattr& attr() const { return attr_; } + + private: + Handler(const Handler&) = delete; + Handler& operator=(const Handler&) = delete; + + FuncPtr handler_; + mutable upb_handlerattr attr_; + mutable bool registered_; + void *cleanup_data_; + upb_handlerfree *cleanup_func_; +}; + +/* A upb::Handlers object represents the set of handlers associated with a + * message in the graph of messages. You can think of it as a big virtual + * table with functions corresponding to all the events that can fire while + * parsing or visiting a message of a specific type. + * + * Any handlers that are not set behave as if they had successfully consumed + * the value. Any unset Start* handlers will propagate their closure to the + * inner frame. + * + * The easiest way to create the *Handler objects needed by the Set* methods is + * with the UpbBind() and UpbMakeHandler() macros; see below. */ +class upb::HandlersPtr { + public: + HandlersPtr(upb_handlers* ptr) : ptr_(ptr) {} + + upb_handlers* ptr() const { return ptr_; } + + typedef upb_selector_t Selector; + typedef upb_handlertype_t Type; + + typedef Handler StartFieldHandler; + typedef Handler EndFieldHandler; + typedef Handler StartMessageHandler; + typedef Handler + EndMessageHandler; + typedef Handler StartStringHandler; + typedef Handler + StringHandler; + + template struct ValueHandler { + typedef Handler H; + }; + + typedef ValueHandler::H Int32Handler; + typedef ValueHandler::H Int64Handler; + typedef ValueHandler::H UInt32Handler; + typedef ValueHandler::H UInt64Handler; + typedef ValueHandler::H FloatHandler; + typedef ValueHandler::H DoubleHandler; + typedef ValueHandler::H BoolHandler; + + /* Any function pointer can be converted to this and converted back to its + * correct type. */ + typedef void GenericFunction(); + + typedef void HandlersCallback(const void *closure, upb_handlers *h); + + /* Returns the msgdef associated with this handlers object. */ + MessageDefPtr message_def() const { + return MessageDefPtr(upb_handlers_msgdef(ptr())); + } + + /* Adds the given pointer and function to the list of cleanup functions that + * will be run when these handlers are freed. If this pointer has previously + * been registered, the function returns false and does nothing. */ + bool AddCleanup(void *ptr, upb_handlerfree *cleanup) { + return upb_handlers_addcleanup(ptr_, ptr, cleanup); + } + + /* Sets the startmsg handler for the message, which is defined as follows: + * + * bool startmsg(MyType* closure) { + * // Called when the message begins. Returns true if processing should + * // continue. + * return true; + * } + */ + bool SetStartMessageHandler(const StartMessageHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartmsg(ptr(), h.handler(), &h.attr()); + } + + /* Sets the endmsg handler for the message, which is defined as follows: + * + * bool endmsg(MyType* closure, upb_status *status) { + * // Called when processing of this message ends, whether in success or + * // failure. "status" indicates the final status of processing, and + * // can also be modified in-place to update the final status. + * } + */ + bool SetEndMessageHandler(const EndMessageHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setendmsg(ptr(), h.handler(), &h.attr()); + } + + /* Sets the value handler for the given field, which is defined as follows + * (this is for an int32 field; other field types will pass their native + * C/C++ type for "val"): + * + * bool OnValue(MyClosure* c, const MyHandlerData* d, int32_t val) { + * // Called when the field's value is encountered. "d" contains + * // whatever data was bound to this field when it was registered. + * // Returns true if processing should continue. + * return true; + * } + * + * handers->SetInt32Handler(f, UpbBind(OnValue, new MyHandlerData(...))); + * + * The value type must exactly match f->type(). + * For example, a handler that takes an int32_t parameter may only be used for + * fields of type UPB_TYPE_INT32 and UPB_TYPE_ENUM. + * + * Returns false if the handler failed to register; in this case the cleanup + * handler (if any) will be called immediately. + */ + bool SetInt32Handler(FieldDefPtr f, const Int32Handler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setint32(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetInt64Handler (FieldDefPtr f, const Int64Handler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setint64(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetUInt32Handler(FieldDefPtr f, const UInt32Handler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setuint32(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetUInt64Handler(FieldDefPtr f, const UInt64Handler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setuint64(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetFloatHandler (FieldDefPtr f, const FloatHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setfloat(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetDoubleHandler(FieldDefPtr f, const DoubleHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setdouble(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetBoolHandler(FieldDefPtr f, const BoolHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setbool(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Like the previous, but templated on the type on the value (ie. int32). + * This is mostly useful to call from other templates. To call this you must + * specify the template parameter explicitly, ie: + * h->SetValueHandler(f, UpbBind(MyHandler, MyData)); */ + template + bool SetValueHandler( + FieldDefPtr f, + const typename ValueHandler::Type>::H &handler); + + /* Sets handlers for a string field, which are defined as follows: + * + * MySubClosure* startstr(MyClosure* c, const MyHandlerData* d, + * size_t size_hint) { + * // Called when a string value begins. The return value indicates the + * // closure for the string. "size_hint" indicates the size of the + * // string if it is known, however if the string is length-delimited + * // and the end-of-string is not available size_hint will be zero. + * // This case is indistinguishable from the case where the size is + * // known to be zero. + * // + * // TODO(haberman): is it important to distinguish these cases? + * // If we had ssize_t as a type we could make -1 "unknown", but + * // ssize_t is POSIX (not ANSI) and therefore less portable. + * // In practice I suspect it won't be important to distinguish. + * return closure; + * } + * + * size_t str(MyClosure* closure, const MyHandlerData* d, + * const char *str, size_t len) { + * // Called for each buffer of string data; the multiple physical buffers + * // are all part of the same logical string. The return value indicates + * // how many bytes were consumed. If this number is less than "len", + * // this will also indicate that processing should be halted for now, + * // like returning false or UPB_BREAK from any other callback. If + * // number is greater than "len", the excess bytes will be skipped over + * // and not passed to the callback. + * return len; + * } + * + * bool endstr(MyClosure* c, const MyHandlerData* d) { + * // Called when a string value ends. Return value indicates whether + * // processing should continue. + * return true; + * } + */ + bool SetStartStringHandler(FieldDefPtr f, const StartStringHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartstr(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetStringHandler(FieldDefPtr f, const StringHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setstring(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetEndStringHandler(FieldDefPtr f, const EndFieldHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setendstr(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Sets the startseq handler, which is defined as follows: + * + * MySubClosure *startseq(MyClosure* c, const MyHandlerData* d) { + * // Called when a sequence (repeated field) begins. The returned + * // pointer indicates the closure for the sequence (or UPB_BREAK + * // to interrupt processing). + * return closure; + * } + * + * h->SetStartSequenceHandler(f, UpbBind(startseq, new MyHandlerData(...))); + * + * Returns "false" if "f" does not belong to this message or is not a + * repeated field. + */ + bool SetStartSequenceHandler(FieldDefPtr f, const StartFieldHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartseq(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Sets the startsubmsg handler for the given field, which is defined as + * follows: + * + * MySubClosure* startsubmsg(MyClosure* c, const MyHandlerData* d) { + * // Called when a submessage begins. The returned pointer indicates the + * // closure for the sequence (or UPB_BREAK to interrupt processing). + * return closure; + * } + * + * h->SetStartSubMessageHandler(f, UpbBind(startsubmsg, + * new MyHandlerData(...))); + * + * Returns "false" if "f" does not belong to this message or is not a + * submessage/group field. + */ + bool SetStartSubMessageHandler(FieldDefPtr f, const StartFieldHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartsubmsg(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Sets the endsubmsg handler for the given field, which is defined as + * follows: + * + * bool endsubmsg(MyClosure* c, const MyHandlerData* d) { + * // Called when a submessage ends. Returns true to continue processing. + * return true; + * } + * + * Returns "false" if "f" does not belong to this message or is not a + * submessage/group field. + */ + bool SetEndSubMessageHandler(FieldDefPtr f, const EndFieldHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setendsubmsg(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Starts the endsubseq handler for the given field, which is defined as + * follows: + * + * bool endseq(MyClosure* c, const MyHandlerData* d) { + * // Called when a sequence ends. Returns true continue processing. + * return true; + * } + * + * Returns "false" if "f" does not belong to this message or is not a + * repeated field. + */ + bool SetEndSequenceHandler(FieldDefPtr f, const EndFieldHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setendseq(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + private: + upb_handlers* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_handlercache ***********************************************************/ + +/* A upb_handlercache lazily builds and caches upb_handlers. You pass it a + * function (with optional closure) that can build handlers for a given + * message on-demand, and the cache maintains a map of msgdef->handlers. */ + +#ifdef __cplusplus +extern "C" { +#endif + +struct upb_handlercache; +typedef struct upb_handlercache upb_handlercache; + +typedef void upb_handlers_callback(const void *closure, upb_handlers *h); + +upb_handlercache *upb_handlercache_new(upb_handlers_callback *callback, + const void *closure); +void upb_handlercache_free(upb_handlercache *cache); +const upb_handlers *upb_handlercache_get(upb_handlercache *cache, + const upb_msgdef *md); +bool upb_handlercache_addcleanup(upb_handlercache *h, void *p, + upb_handlerfree *hfree); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::HandlerCache { + public: + HandlerCache(upb_handlers_callback *callback, const void *closure) + : ptr_(upb_handlercache_new(callback, closure), upb_handlercache_free) {} + HandlerCache(HandlerCache&&) = default; + HandlerCache& operator=(HandlerCache&&) = default; + HandlerCache(upb_handlercache* c) : ptr_(c, upb_handlercache_free) {} + + upb_handlercache* ptr() { return ptr_.get(); } + + const upb_handlers *Get(MessageDefPtr md) { + return upb_handlercache_get(ptr_.get(), md.ptr()); + } + + private: + std::unique_ptr ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_byteshandler ***********************************************************/ + +typedef struct { + upb_func *func; + + /* It is wasteful to include the entire attributes here: + * + * * Some of the information is redundant (like storing the closure type + * separately for each handler that must match). + * * Some of the info is only needed prior to freeze() (like closure types). + * * alignment padding wastes a lot of space for alwaysok_. + * + * If/when the size and locality of handlers is an issue, we can optimize this + * not to store the entire attr like this. We do not expose the table's + * layout to allow this optimization in the future. */ + upb_handlerattr attr; +} upb_handlers_tabent; + +#define UPB_TABENT_INIT {NULL, UPB_HANDLERATTR_INIT} + +typedef struct { + upb_handlers_tabent table[3]; +} upb_byteshandler; + +#define UPB_BYTESHANDLER_INIT \ + { \ + { UPB_TABENT_INIT, UPB_TABENT_INIT, UPB_TABENT_INIT } \ + } + +UPB_INLINE void upb_byteshandler_init(upb_byteshandler *handler) { + upb_byteshandler init = UPB_BYTESHANDLER_INIT; + *handler = init; +} + +#ifdef __cplusplus +extern "C" { +#endif + +/* Caller must ensure that "d" outlives the handlers. */ +bool upb_byteshandler_setstartstr(upb_byteshandler *h, + upb_startstr_handlerfunc *func, void *d); +bool upb_byteshandler_setstring(upb_byteshandler *h, + upb_string_handlerfunc *func, void *d); +bool upb_byteshandler_setendstr(upb_byteshandler *h, + upb_endfield_handlerfunc *func, void *d); + +#ifdef __cplusplus +} /* extern "C" */ + +namespace upb { +typedef upb_byteshandler BytesHandler; +} +#endif + +/** Message handlers ******************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* These are the handlers used internally by upb_msgfactory_getmergehandlers(). + * They write scalar data to a known offset from the message pointer. + * + * These would be trivial for anyone to implement themselves, but it's better + * to use these because some JITs will recognize and specialize these instead + * of actually calling the function. */ + +/* Sets a handler for the given primitive field that will write the data at the + * given offset. If hasbit > 0, also sets a hasbit at the given bit offset + * (addressing each byte low to high). */ +bool upb_msg_setscalarhandler(upb_handlers *h, + const upb_fielddef *f, + size_t offset, + int32_t hasbit); + +/* If the given handler is a msghandlers_primitive field, returns true and sets + * *type, *offset and *hasbit. Otherwise returns false. */ +bool upb_msg_getscalarhandlerdata(const upb_handlers *h, + upb_selector_t s, + upb_fieldtype_t *type, + size_t *offset, + int32_t *hasbit); + + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#include "upb/handlers-inl.h" + +#endif /* UPB_HANDLERS_H */ diff --git a/third_party/upb/upb/json/parser.h b/third_party/upb/upb/json/parser.h new file mode 100644 index 00000000000..d323c52ef8c --- /dev/null +++ b/third_party/upb/upb/json/parser.h @@ -0,0 +1,140 @@ +/* +** upb::json::Parser (upb_json_parser) +** +** Parses JSON according to a specific schema. +** Support for parsing arbitrary JSON (schema-less) will be added later. +*/ + +#ifndef UPB_JSON_PARSER_H_ +#define UPB_JSON_PARSER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace json { +class CodeCache; +class ParserPtr; +class ParserMethodPtr; +} /* namespace json */ +} /* namespace upb */ +#endif + +/* upb_json_parsermethod ******************************************************/ + +struct upb_json_parsermethod; +typedef struct upb_json_parsermethod upb_json_parsermethod; + +#ifdef __cplusplus +extern "C" { +#endif + +const upb_byteshandler* upb_json_parsermethod_inputhandler( + const upb_json_parsermethod* m); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::json::ParserMethodPtr { + public: + ParserMethodPtr() : ptr_(nullptr) {} + ParserMethodPtr(const upb_json_parsermethod* ptr) : ptr_(ptr) {} + + const upb_json_parsermethod* ptr() const { return ptr_; } + + const BytesHandler* input_handler() const { + return upb_json_parsermethod_inputhandler(ptr()); + } + + private: + const upb_json_parsermethod* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_json_parser ************************************************************/ + +/* Preallocation hint: parser won't allocate more bytes than this when first + * constructed. This hint may be an overestimate for some build configurations. + * But if the parser library is upgraded without recompiling the application, + * it may be an underestimate. */ +#define UPB_JSON_PARSER_SIZE 5712 + +struct upb_json_parser; +typedef struct upb_json_parser upb_json_parser; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_json_parser* upb_json_parser_create(upb_arena* a, + const upb_json_parsermethod* m, + const upb_symtab* symtab, + upb_sink output, + upb_status *status, + bool ignore_json_unknown); +upb_bytessink upb_json_parser_input(upb_json_parser* p); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Parses an incoming BytesStream, pushing the results to the destination + * sink. */ +class upb::json::ParserPtr { + public: + ParserPtr(upb_json_parser* ptr) : ptr_(ptr) {} + + static ParserPtr Create(Arena* arena, ParserMethodPtr method, + SymbolTable* symtab, Sink output, Status* status, + bool ignore_json_unknown) { + upb_symtab* symtab_ptr = symtab ? symtab->ptr() : nullptr; + return ParserPtr(upb_json_parser_create( + arena->ptr(), method.ptr(), symtab_ptr, output.sink(), status->ptr(), + ignore_json_unknown)); + } + + BytesSink input() { return upb_json_parser_input(ptr_); } + + private: + upb_json_parser* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_json_codecache *********************************************************/ + +/* Lazily builds and caches decoder methods that will push data to the given + * handlers. The upb_symtab object(s) must outlive this object. */ + +struct upb_json_codecache; +typedef struct upb_json_codecache upb_json_codecache; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_json_codecache *upb_json_codecache_new(void); +void upb_json_codecache_free(upb_json_codecache *cache); +const upb_json_parsermethod* upb_json_codecache_get(upb_json_codecache* cache, + const upb_msgdef* md); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::json::CodeCache { + public: + CodeCache() : ptr_(upb_json_codecache_new(), upb_json_codecache_free) {} + + /* Returns a DecoderMethod that can push data to the given handlers. + * If a suitable method already exists, it will be returned from the cache. */ + ParserMethodPtr Get(MessageDefPtr md) { + return upb_json_codecache_get(ptr_.get(), md.ptr()); + } + + private: + std::unique_ptr ptr_; +}; + +#endif + +#endif /* UPB_JSON_PARSER_H_ */ diff --git a/third_party/upb/upb/json/parser.rl b/third_party/upb/upb/json/parser.rl new file mode 100644 index 00000000000..2641dda31c3 --- /dev/null +++ b/third_party/upb/upb/json/parser.rl @@ -0,0 +1,3017 @@ +/* +** upb::json::Parser (upb_json_parser) +** +** A parser that uses the Ragel State Machine Compiler to generate +** the finite automata. +** +** Ragel only natively handles regular languages, but we can manually +** program it a bit to handle context-free languages like JSON, by using +** the "fcall" and "fret" constructs. +** +** This parser can handle the basics, but needs several things to be fleshed +** out: +** +** - handling of unicode escape sequences (including high surrogate pairs). +** - properly check and report errors for unknown fields, stack overflow, +** improper array nesting (or lack of nesting). +** - handling of base64 sequences with padding characters. +** - handling of push-back (non-success returns from sink functions). +** - handling of keys/escape-sequences/etc that span input buffers. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "upb/json/parser.h" +#include "upb/pb/encoder.h" + +#include "upb/port_def.inc" + +#define UPB_JSON_MAX_DEPTH 64 + +/* Type of value message */ +enum { + VALUE_NULLVALUE = 0, + VALUE_NUMBERVALUE = 1, + VALUE_STRINGVALUE = 2, + VALUE_BOOLVALUE = 3, + VALUE_STRUCTVALUE = 4, + VALUE_LISTVALUE = 5 +}; + +/* Forward declare */ +static bool is_top_level(upb_json_parser *p); +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type); +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type); + +static bool is_number_wrapper_object(upb_json_parser *p); +static bool does_number_wrapper_start(upb_json_parser *p); +static bool does_number_wrapper_end(upb_json_parser *p); + +static bool is_string_wrapper_object(upb_json_parser *p); +static bool does_string_wrapper_start(upb_json_parser *p); +static bool does_string_wrapper_end(upb_json_parser *p); + +static bool does_fieldmask_start(upb_json_parser *p); +static bool does_fieldmask_end(upb_json_parser *p); +static void start_fieldmask_object(upb_json_parser *p); +static void end_fieldmask_object(upb_json_parser *p); + +static void start_wrapper_object(upb_json_parser *p); +static void end_wrapper_object(upb_json_parser *p); + +static void start_value_object(upb_json_parser *p, int value_type); +static void end_value_object(upb_json_parser *p); + +static void start_listvalue_object(upb_json_parser *p); +static void end_listvalue_object(upb_json_parser *p); + +static void start_structvalue_object(upb_json_parser *p); +static void end_structvalue_object(upb_json_parser *p); + +static void start_object(upb_json_parser *p); +static void end_object(upb_json_parser *p); + +static void start_any_object(upb_json_parser *p, const char *ptr); +static bool end_any_object(upb_json_parser *p, const char *ptr); + +static bool start_subobject(upb_json_parser *p); +static void end_subobject(upb_json_parser *p); + +static void start_member(upb_json_parser *p); +static void end_member(upb_json_parser *p); +static bool end_membername(upb_json_parser *p); + +static void start_any_member(upb_json_parser *p, const char *ptr); +static void end_any_member(upb_json_parser *p, const char *ptr); +static bool end_any_membername(upb_json_parser *p); + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle); +static bool end(void *closure, const void *hd); + +static const char eof_ch = 'e'; + +/* stringsink */ +typedef struct { + upb_byteshandler handler; + upb_bytessink sink; + char *ptr; + size_t len, size; +} upb_stringsink; + + +static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) { + upb_stringsink *sink = _sink; + sink->len = 0; + UPB_UNUSED(hd); + UPB_UNUSED(size_hint); + return sink; +} + +static size_t stringsink_string(void *_sink, const void *hd, const char *ptr, + size_t len, const upb_bufhandle *handle) { + upb_stringsink *sink = _sink; + size_t new_size = sink->size; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + while (sink->len + len > new_size) { + new_size *= 2; + } + + if (new_size != sink->size) { + sink->ptr = realloc(sink->ptr, new_size); + sink->size = new_size; + } + + memcpy(sink->ptr + sink->len, ptr, len); + sink->len += len; + + return len; +} + +void upb_stringsink_init(upb_stringsink *sink) { + upb_byteshandler_init(&sink->handler); + upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL); + upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL); + + upb_bytessink_reset(&sink->sink, &sink->handler, sink); + + sink->size = 32; + sink->ptr = malloc(sink->size); + sink->len = 0; +} + +void upb_stringsink_uninit(upb_stringsink *sink) { free(sink->ptr); } + +typedef struct { + /* For encoding Any value field in binary format. */ + upb_handlercache *encoder_handlercache; + upb_stringsink stringsink; + + /* For decoding Any value field in json format. */ + upb_json_codecache *parser_codecache; + upb_sink sink; + upb_json_parser *parser; + + /* Mark the range of uninterpreted values in json input before type url. */ + const char *before_type_url_start; + const char *before_type_url_end; + + /* Mark the range of uninterpreted values in json input after type url. */ + const char *after_type_url_start; +} upb_jsonparser_any_frame; + +typedef struct { + upb_sink sink; + + /* The current message in which we're parsing, and the field whose value we're + * expecting next. */ + const upb_msgdef *m; + const upb_fielddef *f; + + /* The table mapping json name to fielddef for this message. */ + const upb_strtable *name_table; + + /* We are in a repeated-field context. We need this flag to decide whether to + * handle the array as a normal repeated field or a + * google.protobuf.ListValue/google.protobuf.Value. */ + bool is_repeated; + + /* We are in a repeated-field context, ready to emit mapentries as + * submessages. This flag alters the start-of-object (open-brace) behavior to + * begin a sequence of mapentry messages rather than a single submessage. */ + bool is_map; + + /* We are in a map-entry message context. This flag is set when parsing the + * value field of a single map entry and indicates to all value-field parsers + * (subobjects, strings, numbers, and bools) that the map-entry submessage + * should end as soon as the value is parsed. */ + bool is_mapentry; + + /* If |is_map| or |is_mapentry| is true, |mapfield| refers to the parent + * message's map field that we're currently parsing. This differs from |f| + * because |f| is the field in the *current* message (i.e., the map-entry + * message itself), not the parent's field that leads to this map. */ + const upb_fielddef *mapfield; + + /* We are in an Any message context. This flag is set when parsing the Any + * message and indicates to all field parsers (subobjects, strings, numbers, + * and bools) that the parsed field should be serialized as binary data or + * cached (type url not found yet). */ + bool is_any; + + /* The type of packed message in Any. */ + upb_jsonparser_any_frame *any_frame; + + /* True if the field to be parsed is unknown. */ + bool is_unknown_field; +} upb_jsonparser_frame; + +static void init_frame(upb_jsonparser_frame* frame) { + frame->m = NULL; + frame->f = NULL; + frame->name_table = NULL; + frame->is_repeated = false; + frame->is_map = false; + frame->is_mapentry = false; + frame->mapfield = NULL; + frame->is_any = false; + frame->any_frame = NULL; + frame->is_unknown_field = false; +} + +struct upb_json_parser { + upb_arena *arena; + const upb_json_parsermethod *method; + upb_bytessink input_; + + /* Stack to track the JSON scopes we are in. */ + upb_jsonparser_frame stack[UPB_JSON_MAX_DEPTH]; + upb_jsonparser_frame *top; + upb_jsonparser_frame *limit; + + upb_status *status; + + /* Ragel's internal parsing stack for the parsing state machine. */ + int current_state; + int parser_stack[UPB_JSON_MAX_DEPTH]; + int parser_top; + + /* The handle for the current buffer. */ + const upb_bufhandle *handle; + + /* Accumulate buffer. See details in parser.rl. */ + const char *accumulated; + size_t accumulated_len; + char *accumulate_buf; + size_t accumulate_buf_size; + + /* Multi-part text data. See details in parser.rl. */ + int multipart_state; + upb_selector_t string_selector; + + /* Input capture. See details in parser.rl. */ + const char *capture; + + /* Intermediate result of parsing a unicode escape sequence. */ + uint32_t digit; + + /* For resolve type url in Any. */ + const upb_symtab *symtab; + + /* Whether to proceed if unknown field is met. */ + bool ignore_json_unknown; + + /* Cache for parsing timestamp due to base and zone are handled in different + * handlers. */ + struct tm tm; +}; + +static upb_jsonparser_frame* start_jsonparser_frame(upb_json_parser *p) { + upb_jsonparser_frame *inner; + inner = p->top + 1; + init_frame(inner); + return inner; +} + +struct upb_json_codecache { + upb_arena *arena; + upb_inttable methods; /* upb_msgdef* -> upb_json_parsermethod* */ +}; + +struct upb_json_parsermethod { + const upb_json_codecache *cache; + upb_byteshandler input_handler_; + + /* Maps json_name -> fielddef */ + upb_strtable name_table; +}; + +#define PARSER_CHECK_RETURN(x) if (!(x)) return false + +static upb_jsonparser_any_frame *json_parser_any_frame_new( + upb_json_parser *p) { + upb_jsonparser_any_frame *frame; + + frame = upb_arena_malloc(p->arena, sizeof(upb_jsonparser_any_frame)); + + frame->encoder_handlercache = upb_pb_encoder_newcache(); + frame->parser_codecache = upb_json_codecache_new(); + frame->parser = NULL; + frame->before_type_url_start = NULL; + frame->before_type_url_end = NULL; + frame->after_type_url_start = NULL; + + upb_stringsink_init(&frame->stringsink); + + return frame; +} + +static void json_parser_any_frame_set_payload_type( + upb_json_parser *p, + upb_jsonparser_any_frame *frame, + const upb_msgdef *payload_type) { + const upb_handlers *h; + const upb_json_parsermethod *parser_method; + upb_pb_encoder *encoder; + + /* Initialize encoder. */ + h = upb_handlercache_get(frame->encoder_handlercache, payload_type); + encoder = upb_pb_encoder_create(p->arena, h, frame->stringsink.sink); + + /* Initialize parser. */ + parser_method = upb_json_codecache_get(frame->parser_codecache, payload_type); + upb_sink_reset(&frame->sink, h, encoder); + frame->parser = + upb_json_parser_create(p->arena, parser_method, p->symtab, frame->sink, + p->status, p->ignore_json_unknown); +} + +static void json_parser_any_frame_free(upb_jsonparser_any_frame *frame) { + upb_handlercache_free(frame->encoder_handlercache); + upb_json_codecache_free(frame->parser_codecache); + upb_stringsink_uninit(&frame->stringsink); +} + +static bool json_parser_any_frame_has_type_url( + upb_jsonparser_any_frame *frame) { + return frame->parser != NULL; +} + +static bool json_parser_any_frame_has_value_before_type_url( + upb_jsonparser_any_frame *frame) { + return frame->before_type_url_start != frame->before_type_url_end; +} + +static bool json_parser_any_frame_has_value_after_type_url( + upb_jsonparser_any_frame *frame) { + return frame->after_type_url_start != NULL; +} + +static bool json_parser_any_frame_has_value( + upb_jsonparser_any_frame *frame) { + return json_parser_any_frame_has_value_before_type_url(frame) || + json_parser_any_frame_has_value_after_type_url(frame); +} + +static void json_parser_any_frame_set_before_type_url_end( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (frame->parser == NULL) { + frame->before_type_url_end = ptr; + } +} + +static void json_parser_any_frame_set_after_type_url_start_once( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (json_parser_any_frame_has_type_url(frame) && + frame->after_type_url_start == NULL) { + frame->after_type_url_start = ptr; + } +} + +/* Used to signal that a capture has been suspended. */ +static char suspend_capture; + +static upb_selector_t getsel_for_handlertype(upb_json_parser *p, + upb_handlertype_t type) { + upb_selector_t sel; + bool ok = upb_handlers_getselector(p->top->f, type, &sel); + UPB_ASSERT(ok); + return sel; +} + +static upb_selector_t parser_getsel(upb_json_parser *p) { + return getsel_for_handlertype( + p, upb_handlers_getprimitivehandlertype(p->top->f)); +} + +static bool check_stack(upb_json_parser *p) { + if ((p->top + 1) == p->limit) { + upb_status_seterrmsg(p->status, "Nesting too deep"); + return false; + } + + return true; +} + +static void set_name_table(upb_json_parser *p, upb_jsonparser_frame *frame) { + upb_value v; + const upb_json_codecache *cache = p->method->cache; + bool ok; + const upb_json_parsermethod *method; + + ok = upb_inttable_lookupptr(&cache->methods, frame->m, &v); + UPB_ASSERT(ok); + method = upb_value_getconstptr(v); + + frame->name_table = &method->name_table; +} + +/* There are GCC/Clang built-ins for overflow checking which we could start + * using if there was any performance benefit to it. */ + +static bool checked_add(size_t a, size_t b, size_t *c) { + if (SIZE_MAX - a < b) return false; + *c = a + b; + return true; +} + +static size_t saturating_multiply(size_t a, size_t b) { + /* size_t is unsigned, so this is defined behavior even on overflow. */ + size_t ret = a * b; + if (b != 0 && ret / b != a) { + ret = SIZE_MAX; + } + return ret; +} + + +/* Base64 decoding ************************************************************/ + +/* TODO(haberman): make this streaming. */ + +static const signed char b64table[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62/*+*/, -1, -1, -1, 63/*/ */, + 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, + 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, + -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, + 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, + 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1, + -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, + 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/, + 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/, + 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* Returns the table value sign-extended to 32 bits. Knowing that the upper + * bits will be 1 for unrecognized characters makes it easier to check for + * this error condition later (see below). */ +int32_t b64lookup(unsigned char ch) { return b64table[ch]; } + +/* Returns true if the given character is not a valid base64 character or + * padding. */ +bool nonbase64(unsigned char ch) { return b64lookup(ch) == -1 && ch != '='; } + +static bool base64_push(upb_json_parser *p, upb_selector_t sel, const char *ptr, + size_t len) { + const char *limit = ptr + len; + for (; ptr < limit; ptr += 4) { + uint32_t val; + char output[3]; + + if (limit - ptr < 4) { + upb_status_seterrf(p->status, + "Base64 input for bytes field not a multiple of 4: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6 | + b64lookup(ptr[3]); + + /* Test the upper bit; returns true if any of the characters returned -1. */ + if (val & 0x80000000) { + goto otherchar; + } + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + output[2] = val & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 3, NULL); + } + return true; + +otherchar: + if (nonbase64(ptr[0]) || nonbase64(ptr[1]) || nonbase64(ptr[2]) || + nonbase64(ptr[3]) ) { + upb_status_seterrf(p->status, + "Non-base64 characters in bytes field: %s", + upb_fielddef_name(p->top->f)); + return false; + } if (ptr[2] == '=') { + uint32_t val; + char output; + + /* Last group contains only two input bytes, one output byte. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[3] != '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12; + + UPB_ASSERT(!(val & 0x80000000)); + output = val >> 16; + upb_sink_putstring(p->top->sink, sel, &output, 1, NULL); + return true; + } else { + uint32_t val; + char output[2]; + + /* Last group contains only three input bytes, two output bytes. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[2] == '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6; + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 2, NULL); + return true; + } + +badpadding: + upb_status_seterrf(p->status, + "Incorrect base64 padding for field: %s (%.*s)", + upb_fielddef_name(p->top->f), + 4, ptr); + return false; +} + + +/* Accumulate buffer **********************************************************/ + +/* Functionality for accumulating a buffer. + * + * Some parts of the parser need an entire value as a contiguous string. For + * example, to look up a member name in a hash table, or to turn a string into + * a number, the relevant library routines need the input string to be in + * contiguous memory, even if the value spanned two or more buffers in the + * input. These routines handle that. + * + * In the common case we can just point to the input buffer to get this + * contiguous string and avoid any actual copy. So we optimistically begin + * this way. But there are a few cases where we must instead copy into a + * separate buffer: + * + * 1. The string was not contiguous in the input (it spanned buffers). + * + * 2. The string included escape sequences that need to be interpreted to get + * the true value in a contiguous buffer. */ + +static void assert_accumulate_empty(upb_json_parser *p) { + UPB_ASSERT(p->accumulated == NULL); + UPB_ASSERT(p->accumulated_len == 0); +} + +static void accumulate_clear(upb_json_parser *p) { + p->accumulated = NULL; + p->accumulated_len = 0; +} + +/* Used internally by accumulate_append(). */ +static bool accumulate_realloc(upb_json_parser *p, size_t need) { + void *mem; + size_t old_size = p->accumulate_buf_size; + size_t new_size = UPB_MAX(old_size, 128); + while (new_size < need) { + new_size = saturating_multiply(new_size, 2); + } + + mem = upb_arena_realloc(p->arena, p->accumulate_buf, old_size, new_size); + if (!mem) { + upb_status_seterrmsg(p->status, "Out of memory allocating buffer."); + return false; + } + + p->accumulate_buf = mem; + p->accumulate_buf_size = new_size; + return true; +} + +/* Logically appends the given data to the append buffer. + * If "can_alias" is true, we will try to avoid actually copying, but the buffer + * must be valid until the next accumulate_append() call (if any). */ +static bool accumulate_append(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + size_t need; + + if (!p->accumulated && can_alias) { + p->accumulated = buf; + p->accumulated_len = len; + return true; + } + + if (!checked_add(p->accumulated_len, len, &need)) { + upb_status_seterrmsg(p->status, "Integer overflow."); + return false; + } + + if (need > p->accumulate_buf_size && !accumulate_realloc(p, need)) { + return false; + } + + if (p->accumulated != p->accumulate_buf) { + memcpy(p->accumulate_buf, p->accumulated, p->accumulated_len); + p->accumulated = p->accumulate_buf; + } + + memcpy(p->accumulate_buf + p->accumulated_len, buf, len); + p->accumulated_len += len; + return true; +} + +/* Returns a pointer to the data accumulated since the last accumulate_clear() + * call, and writes the length to *len. This with point either to the input + * buffer or a temporary accumulate buffer. */ +static const char *accumulate_getptr(upb_json_parser *p, size_t *len) { + UPB_ASSERT(p->accumulated); + *len = p->accumulated_len; + return p->accumulated; +} + + +/* Mult-part text data ********************************************************/ + +/* When we have text data in the input, it can often come in multiple segments. + * For example, there may be some raw string data followed by an escape + * sequence. The two segments are processed with different logic. Also buffer + * seams in the input can cause multiple segments. + * + * As we see segments, there are two main cases for how we want to process them: + * + * 1. we want to push the captured input directly to string handlers. + * + * 2. we need to accumulate all the parts into a contiguous buffer for further + * processing (field name lookup, string->number conversion, etc). */ + +/* This is the set of states for p->multipart_state. */ +enum { + /* We are not currently processing multipart data. */ + MULTIPART_INACTIVE = 0, + + /* We are processing multipart data by accumulating it into a contiguous + * buffer. */ + MULTIPART_ACCUMULATE = 1, + + /* We are processing multipart data by pushing each part directly to the + * current string handlers. */ + MULTIPART_PUSHEAGERLY = 2 +}; + +/* Start a multi-part text value where we accumulate the data for processing at + * the end. */ +static void multipart_startaccum(upb_json_parser *p) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_ACCUMULATE; +} + +/* Start a multi-part text value where we immediately push text data to a string + * value with the given selector. */ +static void multipart_start(upb_json_parser *p, upb_selector_t sel) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_PUSHEAGERLY; + p->string_selector = sel; +} + +static bool multipart_text(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + switch (p->multipart_state) { + case MULTIPART_INACTIVE: + upb_status_seterrmsg( + p->status, "Internal error: unexpected state MULTIPART_INACTIVE"); + return false; + + case MULTIPART_ACCUMULATE: + if (!accumulate_append(p, buf, len, can_alias)) { + return false; + } + break; + + case MULTIPART_PUSHEAGERLY: { + const upb_bufhandle *handle = can_alias ? p->handle : NULL; + upb_sink_putstring(p->top->sink, p->string_selector, buf, len, handle); + break; + } + } + + return true; +} + +/* Note: this invalidates the accumulate buffer! Call only after reading its + * contents. */ +static void multipart_end(upb_json_parser *p) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_INACTIVE; + accumulate_clear(p); +} + + +/* Input capture **************************************************************/ + +/* Functionality for capturing a region of the input as text. Gracefully + * handles the case where a buffer seam occurs in the middle of the captured + * region. */ + +static void capture_begin(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + UPB_ASSERT(p->capture == NULL); + p->capture = ptr; +} + +static bool capture_end(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->capture); + if (multipart_text(p, p->capture, ptr - p->capture, true)) { + p->capture = NULL; + return true; + } else { + return false; + } +} + +/* This is called at the end of each input buffer (ie. when we have hit a + * buffer seam). If we are in the middle of capturing the input, this + * processes the unprocessed capture region. */ +static void capture_suspend(upb_json_parser *p, const char **ptr) { + if (!p->capture) return; + + if (multipart_text(p, p->capture, *ptr - p->capture, false)) { + /* We use this as a signal that we were in the middle of capturing, and + * that capturing should resume at the beginning of the next buffer. + * + * We can't use *ptr here, because we have no guarantee that this pointer + * will be valid when we resume (if the underlying memory is freed, then + * using the pointer at all, even to compare to NULL, is likely undefined + * behavior). */ + p->capture = &suspend_capture; + } else { + /* Need to back up the pointer to the beginning of the capture, since + * we were not able to actually preserve it. */ + *ptr = p->capture; + } +} + +static void capture_resume(upb_json_parser *p, const char *ptr) { + if (p->capture) { + UPB_ASSERT(p->capture == &suspend_capture); + p->capture = ptr; + } +} + + +/* Callbacks from the parser **************************************************/ + +/* These are the functions called directly from the parser itself. + * We define these in the same order as their declarations in the parser. */ + +static char escape_char(char in) { + switch (in) { + case 'r': return '\r'; + case 't': return '\t'; + case 'n': return '\n'; + case 'f': return '\f'; + case 'b': return '\b'; + case '/': return '/'; + case '"': return '"'; + case '\\': return '\\'; + default: + UPB_ASSERT(0); + return 'x'; + } +} + +static bool escape(upb_json_parser *p, const char *ptr) { + char ch = escape_char(*ptr); + return multipart_text(p, &ch, 1, false); +} + +static void start_hex(upb_json_parser *p) { + p->digit = 0; +} + +static void hexdigit(upb_json_parser *p, const char *ptr) { + char ch = *ptr; + + p->digit <<= 4; + + if (ch >= '0' && ch <= '9') { + p->digit += (ch - '0'); + } else if (ch >= 'a' && ch <= 'f') { + p->digit += ((ch - 'a') + 10); + } else { + UPB_ASSERT(ch >= 'A' && ch <= 'F'); + p->digit += ((ch - 'A') + 10); + } +} + +static bool end_hex(upb_json_parser *p) { + uint32_t codepoint = p->digit; + + /* emit the codepoint as UTF-8. */ + char utf8[3]; /* support \u0000 -- \uFFFF -- need only three bytes. */ + int length = 0; + if (codepoint <= 0x7F) { + utf8[0] = codepoint; + length = 1; + } else if (codepoint <= 0x07FF) { + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x1F) | 0xC0; + length = 2; + } else /* codepoint <= 0xFFFF */ { + utf8[2] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x0F) | 0xE0; + length = 3; + } + /* TODO(haberman): Handle high surrogates: if codepoint is a high surrogate + * we have to wait for the next escape to get the full code point). */ + + return multipart_text(p, utf8, length, false); +} + +static void start_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_number(upb_json_parser *p, const char *ptr) { + if (is_top_level(p)) { + if (is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NUMBERVALUE); + } else { + return false; + } + } else if (does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NUMBERVALUE); + } + + multipart_startaccum(p); + capture_begin(p, ptr); + return true; +} + +static bool parse_number(upb_json_parser *p, bool is_quoted); + +static bool end_number_nontop(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + return parse_number(p, false); +} + +static bool end_number(upb_json_parser *p, const char *ptr) { + if (!end_number_nontop(p, ptr)) { + return false; + } + + if (does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +/* |buf| is NULL-terminated. |buf| itself will never include quotes; + * |is_quoted| tells us whether this text originally appeared inside quotes. */ +static bool parse_number_from_buffer(upb_json_parser *p, const char *buf, + bool is_quoted) { + size_t len = strlen(buf); + const char *bufend = buf + len; + char *end; + upb_fieldtype_t type = upb_fielddef_type(p->top->f); + double val; + double dummy; + double inf = UPB_INFINITY; + + errno = 0; + + if (len == 0 || buf[0] == ' ') { + return false; + } + + /* For integer types, first try parsing with integer-specific routines. + * If these succeed, they will be more accurate for int64/uint64 than + * strtod(). + */ + switch (type) { + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else if (val > INT32_MAX || val < INT32_MIN) { + return false; + } else { + upb_sink_putint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT32: { + unsigned long val = strtoul(buf, &end, 0); + if (end != bufend) { + break; + } else if (val > UINT32_MAX || errno == ERANGE) { + return false; + } else { + upb_sink_putuint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + /* XXX: We can't handle [u]int64 properly on 32-bit machines because + * strto[u]ll isn't in C89. */ + case UPB_TYPE_INT64: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else { + upb_sink_putint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT64: { + unsigned long val = strtoul(p->accumulated, &end, 0); + if (end != bufend) { + break; + } else if (errno == ERANGE) { + return false; + } else { + upb_sink_putuint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + default: + break; + } + + if (type != UPB_TYPE_DOUBLE && type != UPB_TYPE_FLOAT && is_quoted) { + /* Quoted numbers for integer types are not allowed to be in double form. */ + return false; + } + + if (len == strlen("Infinity") && strcmp(buf, "Infinity") == 0) { + /* C89 does not have an INFINITY macro. */ + val = inf; + } else if (len == strlen("-Infinity") && strcmp(buf, "-Infinity") == 0) { + val = -inf; + } else { + val = strtod(buf, &end); + if (errno == ERANGE || end != bufend) { + return false; + } + } + + switch (type) { +#define CASE(capitaltype, smalltype, ctype, min, max) \ + case UPB_TYPE_ ## capitaltype: { \ + if (modf(val, &dummy) != 0 || val > max || val < min) { \ + return false; \ + } else { \ + upb_sink_put ## smalltype(p->top->sink, parser_getsel(p), \ + (ctype)val); \ + return true; \ + } \ + break; \ + } + case UPB_TYPE_ENUM: + CASE(INT32, int32, int32_t, INT32_MIN, INT32_MAX); + CASE(INT64, int64, int64_t, INT64_MIN, INT64_MAX); + CASE(UINT32, uint32, uint32_t, 0, UINT32_MAX); + CASE(UINT64, uint64, uint64_t, 0, UINT64_MAX); +#undef CASE + + case UPB_TYPE_DOUBLE: + upb_sink_putdouble(p->top->sink, parser_getsel(p), val); + return true; + case UPB_TYPE_FLOAT: + if ((val > FLT_MAX || val < -FLT_MAX) && val != inf && val != -inf) { + return false; + } else { + upb_sink_putfloat(p->top->sink, parser_getsel(p), val); + return true; + } + default: + return false; + } +} + +static bool parse_number(upb_json_parser *p, bool is_quoted) { + size_t len; + const char *buf; + + /* strtol() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + if (!multipart_text(p, "\0", 1, false)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (parse_number_from_buffer(p, buf, is_quoted)) { + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "error parsing number: %s", buf); + multipart_end(p); + return false; + } +} + +static bool parser_putbool(upb_json_parser *p, bool val) { + bool ok; + + if (p->top->f == NULL) { + return true; + } + + if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { + upb_status_seterrf(p->status, + "Boolean value specified for non-bool field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + ok = upb_sink_putbool(p->top->sink, parser_getsel(p), val); + UPB_ASSERT(ok); + + return true; +} + +static bool end_bool(upb_json_parser *p, bool val) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_BOOLVALUE); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_BOOLVALUE)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_BOOLVALUE); + } + + if (p->top->is_unknown_field) { + return true; + } + + if (!parser_putbool(p, val)) { + return false; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static bool end_null(upb_json_parser *p) { + const char *zero_ptr = "0"; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + + /* Fill null_value field. */ + multipart_startaccum(p); + capture_begin(p, zero_ptr); + capture_end(p, zero_ptr + 1); + parse_number(p, false); + + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + + return true; +} + +static bool start_any_stringval(upb_json_parser *p) { + multipart_startaccum(p); + return true; +} + +static bool start_stringval(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_string_wrapper_object(p) || + is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + start_fieldmask_object(p); + return true; + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + start_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRINGVALUE); + } else { + return false; + } + } else if (does_string_wrapper_start(p) || + does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (does_fieldmask_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_fieldmask_object(p); + return true; + } else if (is_wellknown_field(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_field(p, UPB_WELLKNOWN_DURATION)) { + if (!start_subobject(p)) { + return false; + } + start_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_STRINGVALUE); + } + + if (p->top->f == NULL) { + multipart_startaccum(p); + return true; + } + + if (p->top->is_any) { + return start_any_stringval(p); + } + + if (upb_fielddef_isstring(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + if (upb_fielddef_type(p->top->f) == UPB_TYPE_STRING) { + /* For STRING fields we push data directly to the handlers as it is + * parsed. We don't do this yet for BYTES fields, because our base64 + * decoder is not streaming. + * + * TODO(haberman): make base64 decoding streaming also. */ + multipart_start(p, getsel_for_handlertype(p, UPB_HANDLER_STRING)); + return true; + } else { + multipart_startaccum(p); + return true; + } + } else if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL && + upb_fielddef_type(p->top->f) != UPB_TYPE_MESSAGE) { + /* No need to push a frame -- numeric values in quotes remain in the + * current parser frame. These values must accmulate so we can convert + * them all at once at the end. */ + multipart_startaccum(p); + return true; + } else { + upb_status_seterrf(p->status, + "String specified for bool or submessage field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool end_any_stringval(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Set type_url */ + upb_selector_t sel; + upb_jsonparser_frame *inner; + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + multipart_end(p); + + /* Resolve type url */ + if (strncmp(buf, "type.googleapis.com/", 20) == 0 && len > 20) { + const upb_msgdef *payload_type = NULL; + buf += 20; + len -= 20; + + payload_type = upb_symtab_lookupmsg2(p->symtab, buf, len); + if (payload_type == NULL) { + upb_status_seterrf( + p->status, "Cannot find packed type: %.*s\n", (int)len, buf); + return false; + } + + json_parser_any_frame_set_payload_type(p, p->top->any_frame, payload_type); + + return true; + } else { + upb_status_seterrf( + p->status, "Invalid type url: %.*s\n", (int)len, buf); + return false; + } +} + +static bool end_stringval_nontop(upb_json_parser *p) { + bool ok = true; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + multipart_end(p); + return true; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_stringval(p); + } + + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_BYTES: + if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + /* Fall through. */ + + case UPB_TYPE_STRING: { + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + break; + } + + case UPB_TYPE_ENUM: { + /* Resolve enum symbolic name to integer value. */ + const upb_enumdef *enumdef = upb_fielddef_enumsubdef(p->top->f); + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + int32_t int_val = 0; + ok = upb_enumdef_ntoi(enumdef, buf, len, &int_val); + + if (ok) { + upb_selector_t sel = parser_getsel(p); + upb_sink_putint32(p->top->sink, sel, int_val); + } else { + upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf); + } + + break; + } + + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + ok = parse_number(p, true); + break; + + default: + UPB_ASSERT(false); + upb_status_seterrmsg(p->status, "Internal error in JSON decoder"); + ok = false; + break; + } + + multipart_end(p); + + return ok; +} + +static bool end_stringval(upb_json_parser *p) { + /* FieldMask's stringvals have been ended when handling them. Only need to + * close FieldMask here.*/ + if (does_fieldmask_end(p)) { + end_fieldmask_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (!end_stringval_nontop(p)) { + return false; + } + + if (does_string_wrapper_end(p) || + does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION) || + is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + end_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static void start_duration_base(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_duration_base(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char seconds_buf[14]; + char nanos_buf[12]; + char *end; + int64_t seconds = 0; + int32_t nanos = 0; + double val = 0.0; + const char *seconds_membername = "seconds"; + const char *nanos_membername = "nanos"; + size_t fraction_start; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + memset(seconds_buf, 0, 14); + memset(nanos_buf, 0, 12); + + /* Find out base end. The maximus duration is 315576000000, which cannot be + * represented by double without losing precision. Thus, we need to handle + * fraction and base separately. */ + for (fraction_start = 0; fraction_start < len && buf[fraction_start] != '.'; + fraction_start++); + + /* Parse base */ + memcpy(seconds_buf, buf, fraction_start); + seconds = strtol(seconds_buf, &end, 10); + if (errno == ERANGE || end != seconds_buf + fraction_start) { + upb_status_seterrf(p->status, "error parsing duration: %s", + seconds_buf); + return false; + } + + if (seconds > 315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "maximum acceptable value is " + "315576000000"); + return false; + } + + if (seconds < -315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "minimum acceptable value is " + "-315576000000"); + return false; + } + + /* Parse fraction */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf + fraction_start, len - fraction_start); + val = strtod(nanos_buf, &end); + if (errno == ERANGE || end != nanos_buf + len - fraction_start + 1) { + upb_status_seterrf(p->status, "error parsing duration: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + if (seconds < 0) nanos = -nanos; + + /* Clean up buffer */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous arena */ + multipart_startaccum(p); + + return true; +} + +static int parse_timestamp_number(upb_json_parser *p) { + size_t len; + const char *buf; + int val; + + /* atoi() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + multipart_text(p, "\0", 1, false); + + buf = accumulate_getptr(p, &len); + val = atoi(buf); + multipart_end(p); + multipart_startaccum(p); + + return val; +} + +static void start_year(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_year(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_year = parse_timestamp_number(p) - 1900; + return true; +} + +static void start_month(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_month(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mon = parse_timestamp_number(p) - 1; + return true; +} + +static void start_day(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_day(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mday = parse_timestamp_number(p); + return true; +} + +static void start_hour(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_hour(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_hour = parse_timestamp_number(p); + return true; +} + +static void start_minute(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_minute(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_min = parse_timestamp_number(p); + return true; +} + +static void start_second(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_second(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_sec = parse_timestamp_number(p); + return true; +} + +static void start_timestamp_base(upb_json_parser *p) { + memset(&p->tm, 0, sizeof(struct tm)); +} + +static void start_timestamp_fraction(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_timestamp_fraction(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char nanos_buf[12]; + char *end; + double val = 0.0; + int32_t nanos; + const char *nanos_membername = "nanos"; + + memset(nanos_buf, 0, 12); + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (len > 10) { + upb_status_seterrf(p->status, + "error parsing timestamp: at most 9-digit fraction."); + return false; + } + + /* Parse nanos */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf, len); + val = strtod(nanos_buf, &end); + + if (errno == ERANGE || end != nanos_buf + len + 1) { + upb_status_seterrf(p->status, "error parsing timestamp nanos: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + + /* Clean up previous environment */ + multipart_end(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_timestamp_zone(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +#define EPOCH_YEAR 1970 +#define TM_YEAR_BASE 1900 + +static bool isleap(int year) { + return (year % 4) == 0 && (year % 100 != 0 || (year % 400) == 0); +} + +const unsigned short int __mon_yday[2][13] = { + /* Normal years. */ + { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, + /* Leap years. */ + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } +}; + +int64_t epoch(int year, int yday, int hour, int min, int sec) { + int64_t years = year - EPOCH_YEAR; + + int64_t leap_days = years / 4 - years / 100 + years / 400; + + int64_t days = years * 365 + yday + leap_days; + int64_t hours = days * 24 + hour; + int64_t mins = hours * 60 + min; + int64_t secs = mins * 60 + sec; + return secs; +} + + +static int64_t upb_mktime(const struct tm *tp) { + int sec = tp->tm_sec; + int min = tp->tm_min; + int hour = tp->tm_hour; + int mday = tp->tm_mday; + int mon = tp->tm_mon; + int year = tp->tm_year + TM_YEAR_BASE; + + /* Calculate day of year from year, month, and day of month. */ + int mon_yday = ((__mon_yday[isleap(year)][mon]) - 1); + int yday = mon_yday + mday; + + return epoch(year, yday, hour, min, sec); +} + +static bool end_timestamp_zone(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + int hours; + int64_t seconds; + const char *seconds_membername = "seconds"; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (buf[0] != 'Z') { + if (sscanf(buf + 1, "%2d:00", &hours) != 1) { + upb_status_seterrf(p->status, "error parsing timestamp offset"); + return false; + } + + if (buf[0] == '+') { + hours = -hours; + } + + p->tm.tm_hour += hours; + } + + /* Normalize tm */ + seconds = upb_mktime(&p->tm); + + /* Check timestamp boundary */ + if (seconds < -62135596800) { + upb_status_seterrf(p->status, "error parsing timestamp: " + "minimum acceptable value is " + "0001-01-01T00:00:00Z"); + return false; + } + + /* Clean up previous environment */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_fieldmask_path(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + multipart_startaccum(p); + return true; +} + +static bool lower_camel_push( + upb_json_parser *p, upb_selector_t sel, const char *ptr, size_t len) { + const char *limit = ptr + len; + bool first = true; + for (;ptr < limit; ptr++) { + if (*ptr >= 'A' && *ptr <= 'Z' && !first) { + char lower = tolower(*ptr); + upb_sink_putstring(p->top->sink, sel, "_", 1, NULL); + upb_sink_putstring(p->top->sink, sel, &lower, 1, NULL); + } else { + upb_sink_putstring(p->top->sink, sel, ptr, 1, NULL); + } + first = false; + } + return true; +} + +static bool end_fieldmask_path(upb_json_parser *p) { + upb_selector_t sel; + + if (!lower_camel_push( + p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + + multipart_end(p); + return true; +} + +static void start_member(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + multipart_startaccum(p); +} + +/* Helper: invoked during parse_mapentry() to emit the mapentry message's key + * field based on the current contents of the accumulate buffer. */ +static bool parse_mapentry_key(upb_json_parser *p) { + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Emit the key field. We do a bit of ad-hoc parsing here because the + * parser state machine has already decided that this is a string field + * name, and we are reinterpreting it as some arbitrary key type. In + * particular, integer and bool keys are quoted, so we need to parse the + * quoted string contents here. */ + + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_KEY); + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no key"); + return false; + } + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + /* Invoke end_number. The accum buffer has the number's text already. */ + if (!parse_number(p, true)) { + return false; + } + break; + case UPB_TYPE_BOOL: + if (len == 4 && !strncmp(buf, "true", 4)) { + if (!parser_putbool(p, true)) { + return false; + } + } else if (len == 5 && !strncmp(buf, "false", 5)) { + if (!parser_putbool(p, false)) { + return false; + } + } else { + upb_status_seterrmsg(p->status, + "Map bool key not 'true' or 'false'"); + return false; + } + multipart_end(p); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + upb_sink subsink; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, len, &subsink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(subsink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(subsink, sel); + multipart_end(p); + break; + } + default: + upb_status_seterrmsg(p->status, "Invalid field type for map key"); + return false; + } + + return true; +} + +/* Helper: emit one map entry (as a submessage in the map field sequence). This + * is invoked from end_membername(), at the end of the map entry's key string, + * with the map key in the accumulate buffer. It parses the key from that + * buffer, emits the handler calls to start the mapentry submessage (setting up + * its subframe in the process), and sets up state in the subframe so that the + * value parser (invoked next) will emit the mapentry's value field and then + * end the mapentry message. */ + +static bool handle_mapentry(upb_json_parser *p) { + const upb_fielddef *mapfield; + const upb_msgdef *mapentrymsg; + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Map entry: p->top->sink is the seq frame, so we need to start a frame + * for the mapentry itself, and then set |f| in that frame so that the map + * value field is parsed, and also set a flag to end the frame after the + * map-entry value is parsed. */ + if (!check_stack(p)) return false; + + mapfield = p->top->mapfield; + mapentrymsg = upb_fielddef_msgsubdef(mapfield); + + inner = start_jsonparser_frame(p); + p->top->f = mapfield; + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = mapentrymsg; + inner->mapfield = mapfield; + + /* Don't set this to true *yet* -- we reuse parsing handlers below to push + * the key field value to the sink, and these handlers will pop the frame + * if they see is_mapentry (when invoked by the parser state machine, they + * would have just seen the map-entry value, not key). */ + inner->is_mapentry = false; + p->top = inner; + + /* send STARTMSG in submsg frame. */ + upb_sink_startmsg(p->top->sink); + + parse_mapentry_key(p); + + /* Set up the value field to receive the map-entry value. */ + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_VALUE); + p->top->is_mapentry = true; /* set up to pop frame after value is parsed. */ + p->top->mapfield = mapfield; + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no value"); + return false; + } + + return true; +} + +static bool end_membername(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + + if (!p->top->m) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_membername(p); + } else if (p->top->is_map) { + return handle_mapentry(p); + } else { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (upb_strtable_lookup2(p->top->name_table, buf, len, &v)) { + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + + return true; + } else if (p->ignore_json_unknown) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + return false; + } + } +} + +static bool end_any_membername(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (len == 5 && strncmp(buf, "@type", len) == 0) { + upb_strtable_lookup2(p->top->name_table, "type_url", 8, &v); + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + return true; + } else { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } +} + +static void end_member(upb_json_parser *p) { + /* If we just parsed a map-entry value, end that frame too. */ + if (p->top->is_mapentry) { + upb_selector_t sel; + bool ok; + const upb_fielddef *mapfield; + + UPB_ASSERT(p->top > p->stack); + /* send ENDMSG on submsg. */ + upb_sink_endmsg(p->top->sink, p->status); + mapfield = p->top->mapfield; + + /* send ENDSUBMSG in repeated-field-of-mapentries frame. */ + p->top--; + ok = upb_handlers_getselector(mapfield, UPB_HANDLER_ENDSUBMSG, &sel); + UPB_ASSERT(ok); + upb_sink_endsubmsg(p->top->sink, sel); + } + + p->top->f = NULL; + p->top->is_unknown_field = false; +} + +static void start_any_member(upb_json_parser *p, const char *ptr) { + start_member(p); + json_parser_any_frame_set_after_type_url_start_once(p->top->any_frame, ptr); +} + +static void end_any_member(upb_json_parser *p, const char *ptr) { + json_parser_any_frame_set_before_type_url_end(p->top->any_frame, ptr); + end_member(p); +} + +static bool start_subobject(upb_json_parser *p) { + if (p->top->is_unknown_field) { + if (!check_stack(p)) return false; + + p->top = start_jsonparser_frame(p); + return true; + } + + if (upb_fielddef_ismap(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a map. Start a new parser frame in a repeated-field + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->mapfield = p->top->f; + inner->is_map = true; + p->top = inner; + + return true; + } else if (upb_fielddef_issubmsg(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a subobject. Start a new parser frame in the submsg + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + set_name_table(p, inner); + p->top = inner; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + + return true; + } else { + upb_status_seterrf(p->status, + "Object specified for non-message/group field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool start_subobject_full(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + start_structvalue_object(p); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_STRUCT)) { + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } + + return start_subobject(p); +} + +static void end_subobject(upb_json_parser *p) { + if (is_top_level(p)) { + return; + } + + if (p->top->is_map) { + upb_selector_t sel; + p->top--; + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + } else { + upb_selector_t sel; + bool is_unknown = p->top->m == NULL; + p->top--; + if (!is_unknown) { + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(p->top->sink, sel); + } + } +} + +static void end_subobject_full(upb_json_parser *p) { + end_subobject(p); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + end_structvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static bool start_array(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + start_listvalue_object(p); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_LISTVALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } + + if (p->top->is_unknown_field) { + inner = start_jsonparser_frame(p); + inner->is_unknown_field = true; + p->top = inner; + + return true; + } + + if (!upb_fielddef_isseq(p->top->f)) { + upb_status_seterrf(p->status, + "Array specified for non-repeated field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + inner->is_repeated = true; + p->top = inner; + + return true; +} + +static void end_array(upb_json_parser *p) { + upb_selector_t sel; + + UPB_ASSERT(p->top > p->stack); + + p->top--; + + if (p->top->is_unknown_field) { + return; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + end_listvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static void start_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_startmsg(p->top->sink); + } +} + +static void end_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_endmsg(p->top->sink, p->status); + } +} + +static void start_any_object(upb_json_parser *p, const char *ptr) { + start_object(p); + p->top->any_frame->before_type_url_start = ptr; + p->top->any_frame->before_type_url_end = ptr; +} + +static bool end_any_object(upb_json_parser *p, const char *ptr) { + const char *value_membername = "value"; + bool is_well_known_packed = false; + const char *packed_end = ptr + 1; + upb_selector_t sel; + upb_jsonparser_frame *inner; + + if (json_parser_any_frame_has_value(p->top->any_frame) && + !json_parser_any_frame_has_type_url(p->top->any_frame)) { + upb_status_seterrmsg(p->status, "No valid type url"); + return false; + } + + /* Well known types data is represented as value field. */ + if (upb_msgdef_wellknowntype(p->top->any_frame->parser->top->m) != + UPB_WELLKNOWN_UNSPECIFIED) { + is_well_known_packed = true; + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + p->top->any_frame->before_type_url_start = + memchr(p->top->any_frame->before_type_url_start, ':', + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start); + if (p->top->any_frame->before_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "invalid data for well known type."); + return false; + } + p->top->any_frame->before_type_url_start++; + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + p->top->any_frame->after_type_url_start = + memchr(p->top->any_frame->after_type_url_start, ':', + (ptr + 1) - + p->top->any_frame->after_type_url_start); + if (p->top->any_frame->after_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "Invalid data for well known type."); + return false; + } + p->top->any_frame->after_type_url_start++; + packed_end = ptr; + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->before_type_url_start, + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "{", 1, NULL)) { + return false; + } + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame) && + json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, ",", 1, NULL)) { + return false; + } + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->after_type_url_start, + packed_end - p->top->any_frame->after_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "}", 1, NULL)) { + return false; + } + } + } + + if (!end(p->top->any_frame->parser, NULL)) { + return false; + } + + p->top->is_any = false; + + /* Set value */ + start_member(p); + capture_begin(p, value_membername); + capture_end(p, value_membername + 5); + end_membername(p); + + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, p->top->any_frame->stringsink.ptr, + p->top->any_frame->stringsink.len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + end_member(p); + + end_object(p); + + /* Deallocate any parse frame. */ + json_parser_any_frame_free(p->top->any_frame); + + return true; +} + +static bool is_string_wrapper(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_STRINGVALUE || + type == UPB_WELLKNOWN_BYTESVALUE; +} + +static bool is_fieldmask(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_FIELDMASK; +} + +static void start_fieldmask_object(upb_json_parser *p) { + const char *membername = "paths"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); + + start_array(p); +} + +static void end_fieldmask_object(upb_json_parser *p) { + end_array(p); + end_member(p); + end_object(p); +} + +static void start_wrapper_object(upb_json_parser *p) { + const char *membername = "value"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); +} + +static void end_wrapper_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_value_object(upb_json_parser *p, int value_type) { + const char *nullmember = "null_value"; + const char *numbermember = "number_value"; + const char *stringmember = "string_value"; + const char *boolmember = "bool_value"; + const char *structmember = "struct_value"; + const char *listmember = "list_value"; + const char *membername = ""; + + switch (value_type) { + case VALUE_NULLVALUE: + membername = nullmember; + break; + case VALUE_NUMBERVALUE: + membername = numbermember; + break; + case VALUE_STRINGVALUE: + membername = stringmember; + break; + case VALUE_BOOLVALUE: + membername = boolmember; + break; + case VALUE_STRUCTVALUE: + membername = structmember; + break; + case VALUE_LISTVALUE: + membername = listmember; + break; + } + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_value_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_listvalue_object(upb_json_parser *p) { + const char *membername = "values"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_listvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_structvalue_object(upb_json_parser *p) { + const char *membername = "fields"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_structvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static bool is_top_level(upb_json_parser *p) { + return p->top == p->stack && p->top->f == NULL && !p->top->is_unknown_field; +} + +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->m != NULL && upb_msgdef_wellknowntype(p->top->m) == type; +} + +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + (upb_msgdef_wellknowntype(upb_fielddef_msgsubdef(p->top->f)) + == type); +} + +static bool does_number_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + upb_msgdef_isnumberwrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_number_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool is_number_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool does_string_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_string_wrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_string_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool is_string_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool does_fieldmask_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_fieldmask(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_fieldmask_end(upb_json_parser *p) { + return p->top->m != NULL && is_fieldmask(p->top->m); +} + +#define CHECK_RETURN_TOP(x) if (!(x)) goto error + + +/* The actual parser **********************************************************/ + +/* What follows is the Ragel parser itself. The language is specified in Ragel + * and the actions call our C functions above. + * + * Ragel has an extensive set of functionality, and we use only a small part of + * it. There are many action types but we only use a few: + * + * ">" -- transition into a machine + * "%" -- transition out of a machine + * "@" -- transition into a final state of a machine. + * + * "@" transitions are tricky because a machine can transition into a final + * state repeatedly. But in some cases we know this can't happen, for example + * a string which is delimited by a final '"' can only transition into its + * final state once, when the closing '"' is seen. */ + +%%{ + machine json; + + ws = space*; + + integer = "0" | /[1-9]/ /[0-9]/*; + decimal = "." /[0-9]/+; + exponent = /[eE]/ /[+\-]/? /[0-9]/+; + + number_machine := + ("-"? integer decimal? exponent?) + %/{ fhold; fret; } + <: any + >{ fhold; fret; } + ; + number = /[0-9\-]/ >{ fhold; fcall number_machine; }; + + text = + /[^\\"]/+ + >{ start_text(parser, p); } + %{ CHECK_RETURN_TOP(end_text(parser, p)); } + ; + + unicode_char = + "\\u" + /[0-9A-Fa-f]/{4} + >{ start_hex(parser); } + ${ hexdigit(parser, p); } + %{ CHECK_RETURN_TOP(end_hex(parser)); } + ; + + escape_char = + "\\" + /[rtbfn"\/\\]/ + >{ CHECK_RETURN_TOP(escape(parser, p)); } + ; + + string_machine := + (text | unicode_char | escape_char)** + '"' + @{ fhold; fret; } + ; + + year = + (digit digit digit digit) + >{ start_year(parser, p); } + %{ CHECK_RETURN_TOP(end_year(parser, p)); } + ; + month = + (digit digit) + >{ start_month(parser, p); } + %{ CHECK_RETURN_TOP(end_month(parser, p)); } + ; + day = + (digit digit) + >{ start_day(parser, p); } + %{ CHECK_RETURN_TOP(end_day(parser, p)); } + ; + hour = + (digit digit) + >{ start_hour(parser, p); } + %{ CHECK_RETURN_TOP(end_hour(parser, p)); } + ; + minute = + (digit digit) + >{ start_minute(parser, p); } + %{ CHECK_RETURN_TOP(end_minute(parser, p)); } + ; + second = + (digit digit) + >{ start_second(parser, p); } + %{ CHECK_RETURN_TOP(end_second(parser, p)); } + ; + + duration_machine := + ("-"? integer decimal?) + >{ start_duration_base(parser, p); } + %{ CHECK_RETURN_TOP(end_duration_base(parser, p)); } + 's"' + @{ fhold; fret; } + ; + + timestamp_machine := + (year "-" month "-" day "T" hour ":" minute ":" second) + >{ start_timestamp_base(parser); } + ("." digit+)? + >{ start_timestamp_fraction(parser, p); } + %{ CHECK_RETURN_TOP(end_timestamp_fraction(parser, p)); } + ([+\-] digit digit ":00" | "Z") + >{ start_timestamp_zone(parser, p); } + %{ CHECK_RETURN_TOP(end_timestamp_zone(parser, p)); } + '"' + @{ fhold; fret; } + ; + + fieldmask_path_text = + /[^",]/+ + >{ start_fieldmask_path_text(parser, p); } + %{ end_fieldmask_path_text(parser, p); } + ; + + fieldmask_path = + fieldmask_path_text + >{ start_fieldmask_path(parser); } + %{ end_fieldmask_path(parser); } + ; + + fieldmask_machine := + (fieldmask_path ("," fieldmask_path)*)? + '"' + @{ fhold; fret; } + ; + + string = + '"' + @{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { + fcall timestamp_machine; + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { + fcall duration_machine; + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_FIELDMASK)) { + fcall fieldmask_machine; + } else { + fcall string_machine; + } + } + '"'; + + value2 = ^(space | "]" | "}") >{ fhold; fcall value_machine; } ; + + member = + ws + string + >{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_member(parser, p); + } else { + start_member(parser); + } + } + @{ CHECK_RETURN_TOP(end_membername(parser)); } + ws ":" ws + value2 + %{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + end_any_member(parser, p); + } else { + end_member(parser); + } + } + ws; + + object = + ("{" ws) + >{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_object(parser, p); + } else { + start_object(parser); + } + } + (member ("," member)*)? + "}" + >{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + CHECK_RETURN_TOP(end_any_object(parser, p)); + } else { + end_object(parser); + } + } + ; + + element = ws value2 ws; + array = + "[" + >{ CHECK_RETURN_TOP(start_array(parser)); } + ws + (element ("," element)*)? + "]" + >{ end_array(parser); } + ; + + value = + number + >{ CHECK_RETURN_TOP(start_number(parser, p)); } + %{ CHECK_RETURN_TOP(end_number(parser, p)); } + | string + >{ CHECK_RETURN_TOP(start_stringval(parser)); } + @{ CHECK_RETURN_TOP(end_stringval(parser)); } + | "true" + %{ CHECK_RETURN_TOP(end_bool(parser, true)); } + | "false" + %{ CHECK_RETURN_TOP(end_bool(parser, false)); } + | "null" + %{ CHECK_RETURN_TOP(end_null(parser)); } + | object + >{ CHECK_RETURN_TOP(start_subobject_full(parser)); } + %{ end_subobject_full(parser); } + | array; + + value_machine := + value + <: any >{ fhold; fret; } ; + + main := ws value ws; +}%% + +%% write data noerror nofinal; + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle) { + upb_json_parser *parser = closure; + + /* Variables used by Ragel's generated code. */ + int cs = parser->current_state; + int *stack = parser->parser_stack; + int top = parser->parser_top; + + const char *p = buf; + const char *pe = buf + size; + const char *eof = &eof_ch; + + parser->handle = handle; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + capture_resume(parser, buf); + + %% write exec; + + if (p != pe) { + upb_status_seterrf(parser->status, "Parse error at '%.*s'\n", pe - p, p); + } else { + capture_suspend(parser, &p); + } + +error: + /* Save parsing state back to parser. */ + parser->current_state = cs; + parser->parser_top = top; + + return p - buf; +} + +static bool end(void *closure, const void *hd) { + upb_json_parser *parser = closure; + + /* Prevent compile warning on unused static constants. */ + UPB_UNUSED(json_start); + UPB_UNUSED(json_en_duration_machine); + UPB_UNUSED(json_en_fieldmask_machine); + UPB_UNUSED(json_en_number_machine); + UPB_UNUSED(json_en_string_machine); + UPB_UNUSED(json_en_timestamp_machine); + UPB_UNUSED(json_en_value_machine); + UPB_UNUSED(json_en_main); + + parse(parser, hd, &eof_ch, 0, NULL); + + return parser->current_state >= %%{ write first_final; }%%; +} + +static void json_parser_reset(upb_json_parser *p) { + int cs; + int top; + + p->top = p->stack; + init_frame(p->top); + + /* Emit Ragel initialization of the parser. */ + %% write init; + p->current_state = cs; + p->parser_top = top; + accumulate_clear(p); + p->multipart_state = MULTIPART_INACTIVE; + p->capture = NULL; + p->accumulated = NULL; +} + +static upb_json_parsermethod *parsermethod_new(upb_json_codecache *c, + const upb_msgdef *md) { + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + upb_json_parsermethod *m = upb_malloc(alloc, sizeof(*m)); + + m->cache = c; + + upb_byteshandler_init(&m->input_handler_); + upb_byteshandler_setstring(&m->input_handler_, parse, m); + upb_byteshandler_setendstr(&m->input_handler_, end, m); + + upb_strtable_init2(&m->name_table, UPB_CTYPE_CONSTPTR, alloc); + + /* Build name_table */ + + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_value v = upb_value_constptr(f); + char *buf; + + /* Add an entry for the JSON name. */ + size_t len = upb_fielddef_getjsonname(f, NULL, 0); + buf = upb_malloc(alloc, len); + upb_fielddef_getjsonname(f, buf, len); + upb_strtable_insert3(&m->name_table, buf, strlen(buf), v, alloc); + + if (strcmp(buf, upb_fielddef_name(f)) != 0) { + /* Since the JSON name is different from the regular field name, add an + * entry for the raw name (compliant proto3 JSON parsers must accept + * both). */ + const char *name = upb_fielddef_name(f); + upb_strtable_insert3(&m->name_table, name, strlen(name), v, alloc); + } + } + + return m; +} + +/* Public API *****************************************************************/ + +upb_json_parser *upb_json_parser_create(upb_arena *arena, + const upb_json_parsermethod *method, + const upb_symtab* symtab, + upb_sink output, + upb_status *status, + bool ignore_json_unknown) { +#ifndef NDEBUG + const size_t size_before = upb_arena_bytesallocated(arena); +#endif + upb_json_parser *p = upb_arena_malloc(arena, sizeof(upb_json_parser)); + if (!p) return false; + + p->arena = arena; + p->method = method; + p->status = status; + p->limit = p->stack + UPB_JSON_MAX_DEPTH; + p->accumulate_buf = NULL; + p->accumulate_buf_size = 0; + upb_bytessink_reset(&p->input_, &method->input_handler_, p); + + json_parser_reset(p); + p->top->sink = output; + p->top->m = upb_handlers_msgdef(output.handlers); + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + set_name_table(p, p->top); + p->symtab = symtab; + + p->ignore_json_unknown = ignore_json_unknown; + + /* If this fails, uncomment and increase the value in parser.h. */ + /* fprintf(stderr, "%zd\n", upb_arena_bytesallocated(arena) - size_before); */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= + UPB_JSON_PARSER_SIZE); + return p; +} + +upb_bytessink upb_json_parser_input(upb_json_parser *p) { + return p->input_; +} + +const upb_byteshandler *upb_json_parsermethod_inputhandler( + const upb_json_parsermethod *m) { + return &m->input_handler_; +} + +upb_json_codecache *upb_json_codecache_new(void) { + upb_alloc *alloc; + upb_json_codecache *c; + + c = upb_gmalloc(sizeof(*c)); + + c->arena = upb_arena_new(); + alloc = upb_arena_alloc(c->arena); + + upb_inttable_init2(&c->methods, UPB_CTYPE_CONSTPTR, alloc); + + return c; +} + +void upb_json_codecache_free(upb_json_codecache *c) { + upb_arena_free(c->arena); + upb_gfree(c); +} + +const upb_json_parsermethod *upb_json_codecache_get(upb_json_codecache *c, + const upb_msgdef *md) { + upb_json_parsermethod *m; + upb_value v; + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + if (upb_inttable_lookupptr(&c->methods, md, &v)) { + return upb_value_getconstptr(v); + } + + m = parsermethod_new(c, md); + v = upb_value_constptr(m); + + if (!m) return NULL; + if (!upb_inttable_insertptr2(&c->methods, md, v, alloc)) return NULL; + + /* Populate parser methods for all submessages, so the name tables will + * be available during parsing. */ + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + + if (upb_fielddef_issubmsg(f)) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + const upb_json_parsermethod *sub_method = + upb_json_codecache_get(c, subdef); + + if (!sub_method) return NULL; + } + } + + return m; +} diff --git a/third_party/upb/upb/json/printer.c b/third_party/upb/upb/json/printer.c new file mode 100644 index 00000000000..38f817d4d40 --- /dev/null +++ b/third_party/upb/upb/json/printer.c @@ -0,0 +1,1406 @@ +/* +** This currently uses snprintf() to format primitives, and could be optimized +** further. +*/ + +#include "upb/json/printer.h" + +#include +#include +#include +#include + +#include "upb/port_def.inc" + +struct upb_json_printer { + upb_sink input_; + /* BytesSink closure. */ + void *subc_; + upb_bytessink output_; + + /* We track the depth so that we know when to emit startstr/endstr on the + * output. */ + int depth_; + + /* Have we emitted the first element? This state is necessary to emit commas + * without leaving a trailing comma in arrays/maps. We keep this state per + * frame depth. + * + * Why max_depth * 2? UPB_MAX_HANDLER_DEPTH counts depth as nested messages. + * We count frames (contexts in which we separate elements by commas) as both + * repeated fields and messages (maps), and the worst case is a + * message->repeated field->submessage->repeated field->... nesting. */ + bool first_elem_[UPB_MAX_HANDLER_DEPTH * 2]; + + /* To print timestamp, printer needs to cache its seconds and nanos values + * and convert them when ending timestamp message. See comments of + * printer_sethandlers_timestamp for more detail. */ + int64_t seconds; + int32_t nanos; +}; + +/* StringPiece; a pointer plus a length. */ +typedef struct { + char *ptr; + size_t len; +} strpc; + +void freestrpc(void *ptr) { + strpc *pc = ptr; + upb_gfree(pc->ptr); + upb_gfree(pc); +} + +typedef struct { + bool preserve_fieldnames; +} upb_json_printercache; + +/* Convert fielddef name to JSON name and return as a string piece. */ +strpc *newstrpc(upb_handlers *h, const upb_fielddef *f, + bool preserve_fieldnames) { + /* TODO(haberman): handle malloc failure. */ + strpc *ret = upb_gmalloc(sizeof(*ret)); + if (preserve_fieldnames) { + ret->ptr = upb_gstrdup(upb_fielddef_name(f)); + ret->len = strlen(ret->ptr); + } else { + size_t len; + ret->len = upb_fielddef_getjsonname(f, NULL, 0); + ret->ptr = upb_gmalloc(ret->len); + len = upb_fielddef_getjsonname(f, ret->ptr, ret->len); + UPB_ASSERT(len == ret->len); + ret->len--; /* NULL */ + } + + upb_handlers_addcleanup(h, ret, freestrpc); + return ret; +} + +/* Convert a null-terminated const char* to a string piece. */ +strpc *newstrpc_str(upb_handlers *h, const char * str) { + strpc * ret = upb_gmalloc(sizeof(*ret)); + ret->ptr = upb_gstrdup(str); + ret->len = strlen(str); + upb_handlers_addcleanup(h, ret, freestrpc); + return ret; +} + +/* ------------ JSON string printing: values, maps, arrays ------------------ */ + +static void print_data( + upb_json_printer *p, const char *buf, unsigned int len) { + /* TODO: Will need to change if we support pushback from the sink. */ + size_t n = upb_bytessink_putbuf(p->output_, p->subc_, buf, len, NULL); + UPB_ASSERT(n == len); +} + +static void print_comma(upb_json_printer *p) { + if (!p->first_elem_[p->depth_]) { + print_data(p, ",", 1); + } + p->first_elem_[p->depth_] = false; +} + +/* Helpers that print properly formatted elements to the JSON output stream. */ + +/* Used for escaping control chars in strings. */ +static const char kControlCharLimit = 0x20; + +UPB_INLINE bool is_json_escaped(char c) { + /* See RFC 4627. */ + unsigned char uc = (unsigned char)c; + return uc < kControlCharLimit || uc == '"' || uc == '\\'; +} + +UPB_INLINE const char* json_nice_escape(char c) { + switch (c) { + case '"': return "\\\""; + case '\\': return "\\\\"; + case '\b': return "\\b"; + case '\f': return "\\f"; + case '\n': return "\\n"; + case '\r': return "\\r"; + case '\t': return "\\t"; + default: return NULL; + } +} + +/* Write a properly escaped string chunk. The surrounding quotes are *not* + * printed; this is so that the caller has the option of emitting the string + * content in chunks. */ +static void putstring(upb_json_printer *p, const char *buf, unsigned int len) { + const char* unescaped_run = NULL; + unsigned int i; + for (i = 0; i < len; i++) { + char c = buf[i]; + /* Handle escaping. */ + if (is_json_escaped(c)) { + /* Use a "nice" escape, like \n, if one exists for this character. */ + const char* escape = json_nice_escape(c); + /* If we don't have a specific 'nice' escape code, use a \uXXXX-style + * escape. */ + char escape_buf[8]; + if (!escape) { + unsigned char byte = (unsigned char)c; + _upb_snprintf(escape_buf, sizeof(escape_buf), "\\u%04x", (int)byte); + escape = escape_buf; + } + + /* N.B. that we assume that the input encoding is equal to the output + * encoding (both UTF-8 for now), so for chars >= 0x20 and != \, ", we + * can simply pass the bytes through. */ + + /* If there's a current run of unescaped chars, print that run first. */ + if (unescaped_run) { + print_data(p, unescaped_run, &buf[i] - unescaped_run); + unescaped_run = NULL; + } + /* Then print the escape code. */ + print_data(p, escape, strlen(escape)); + } else { + /* Add to the current unescaped run of characters. */ + if (unescaped_run == NULL) { + unescaped_run = &buf[i]; + } + } + } + + /* If the string ended in a run of unescaped characters, print that last run. */ + if (unescaped_run) { + print_data(p, unescaped_run, &buf[len] - unescaped_run); + } +} + +#define CHKLENGTH(x) if (!(x)) return -1; + +/* Helpers that format floating point values according to our custom formats. + * Right now we use %.8g and %.17g for float/double, respectively, to match + * proto2::util::JsonFormat's defaults. May want to change this later. */ + +const char neginf[] = "\"-Infinity\""; +const char inf[] = "\"Infinity\""; + +static size_t fmt_double(double val, char* buf, size_t length) { + if (val == UPB_INFINITY) { + CHKLENGTH(length >= strlen(inf)); + strcpy(buf, inf); + return strlen(inf); + } else if (val == -UPB_INFINITY) { + CHKLENGTH(length >= strlen(neginf)); + strcpy(buf, neginf); + return strlen(neginf); + } else { + size_t n = _upb_snprintf(buf, length, "%.17g", val); + CHKLENGTH(n > 0 && n < length); + return n; + } +} + +static size_t fmt_float(float val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%.8g", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_bool(bool val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%s", (val ? "true" : "false")); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_int64_as_number(long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%lld", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_uint64_as_number( + unsigned long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%llu", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_int64_as_string(long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "\"%lld\"", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_uint64_as_string( + unsigned long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "\"%llu\"", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +/* Print a map key given a field name. Called by scalar field handlers and by + * startseq for repeated fields. */ +static bool putkey(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + const strpc *key = handler_data; + print_comma(p); + print_data(p, "\"", 1); + putstring(p, key->ptr, key->len); + print_data(p, "\":", 2); + return true; +} + +#define CHKFMT(val) if ((val) == (size_t)-1) return false; +#define CHK(val) if (!(val)) return false; + +#define TYPE_HANDLERS(type, fmt_func) \ + static bool put##type(void *closure, const void *handler_data, type val) { \ + upb_json_printer *p = closure; \ + char data[64]; \ + size_t length = fmt_func(val, data, sizeof(data)); \ + UPB_UNUSED(handler_data); \ + CHKFMT(length); \ + print_data(p, data, length); \ + return true; \ + } \ + static bool scalar_##type(void *closure, const void *handler_data, \ + type val) { \ + CHK(putkey(closure, handler_data)); \ + CHK(put##type(closure, handler_data, val)); \ + return true; \ + } \ + static bool repeated_##type(void *closure, const void *handler_data, \ + type val) { \ + upb_json_printer *p = closure; \ + print_comma(p); \ + CHK(put##type(closure, handler_data, val)); \ + return true; \ + } + +#define TYPE_HANDLERS_MAPKEY(type, fmt_func) \ + static bool putmapkey_##type(void *closure, const void *handler_data, \ + type val) { \ + upb_json_printer *p = closure; \ + char data[64]; \ + size_t length = fmt_func(val, data, sizeof(data)); \ + UPB_UNUSED(handler_data); \ + print_data(p, "\"", 1); \ + print_data(p, data, length); \ + print_data(p, "\":", 2); \ + return true; \ + } + +TYPE_HANDLERS(double, fmt_double) +TYPE_HANDLERS(float, fmt_float) +TYPE_HANDLERS(bool, fmt_bool) +TYPE_HANDLERS(int32_t, fmt_int64_as_number) +TYPE_HANDLERS(uint32_t, fmt_int64_as_number) +TYPE_HANDLERS(int64_t, fmt_int64_as_string) +TYPE_HANDLERS(uint64_t, fmt_uint64_as_string) + +/* double and float are not allowed to be map keys. */ +TYPE_HANDLERS_MAPKEY(bool, fmt_bool) +TYPE_HANDLERS_MAPKEY(int32_t, fmt_int64_as_number) +TYPE_HANDLERS_MAPKEY(uint32_t, fmt_int64_as_number) +TYPE_HANDLERS_MAPKEY(int64_t, fmt_int64_as_number) +TYPE_HANDLERS_MAPKEY(uint64_t, fmt_uint64_as_number) + +#undef TYPE_HANDLERS +#undef TYPE_HANDLERS_MAPKEY + +typedef struct { + void *keyname; + const upb_enumdef *enumdef; +} EnumHandlerData; + +static bool scalar_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + const char *symbolic_name; + + CHK(putkey(closure, hd->keyname)); + + symbolic_name = upb_enumdef_iton(hd->enumdef, val); + if (symbolic_name) { + print_data(p, "\"", 1); + putstring(p, symbolic_name, strlen(symbolic_name)); + print_data(p, "\"", 1); + } else { + putint32_t(closure, NULL, val); + } + + return true; +} + +static void print_enum_symbolic_name(upb_json_printer *p, + const upb_enumdef *def, + int32_t val) { + const char *symbolic_name = upb_enumdef_iton(def, val); + if (symbolic_name) { + print_data(p, "\"", 1); + putstring(p, symbolic_name, strlen(symbolic_name)); + print_data(p, "\"", 1); + } else { + putint32_t(p, NULL, val); + } +} + +static bool repeated_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + print_comma(p); + + print_enum_symbolic_name(p, hd->enumdef, val); + + return true; +} + +static bool mapvalue_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + + print_enum_symbolic_name(p, hd->enumdef, val); + + return true; +} + +static void *scalar_startsubmsg(void *closure, const void *handler_data) { + return putkey(closure, handler_data) ? closure : UPB_BREAK; +} + +static void *repeated_startsubmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_comma(p); + return closure; +} + +static void start_frame(upb_json_printer *p) { + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); +} + +static void end_frame(upb_json_printer *p) { + print_data(p, "}", 1); + p->depth_--; +} + +static bool printer_startmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + start_frame(p); + return true; +} + +static bool printer_endmsg(void *closure, const void *handler_data, upb_status *s) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + end_frame(p); + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +static void *startseq(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + CHK(putkey(closure, handler_data)); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "[", 1); + return closure; +} + +static bool endseq(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "]", 1); + p->depth_--; + return true; +} + +static void *startmap(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + CHK(putkey(closure, handler_data)); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); + return closure; +} + +static bool endmap(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "}", 1); + p->depth_--; + return true; +} + +static size_t putstr(void *closure, const void *handler_data, const char *str, + size_t len, const upb_bufhandle *handle) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(handle); + putstring(p, str, len); + return len; +} + +/* This has to Base64 encode the bytes, because JSON has no "bytes" type. */ +static size_t putbytes(void *closure, const void *handler_data, const char *str, + size_t len, const upb_bufhandle *handle) { + upb_json_printer *p = closure; + + /* This is the regular base64, not the "web-safe" version. */ + static const char base64[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /* Base64-encode. */ + char data[16000]; + const char *limit = data + sizeof(data); + const unsigned char *from = (const unsigned char*)str; + char *to = data; + size_t remaining = len; + size_t bytes; + + UPB_UNUSED(handler_data); + UPB_UNUSED(handle); + + print_data(p, "\"", 1); + + while (remaining > 2) { + if (limit - to < 4) { + bytes = to - data; + putstring(p, data, bytes); + to = data; + } + + to[0] = base64[from[0] >> 2]; + to[1] = base64[((from[0] & 0x3) << 4) | (from[1] >> 4)]; + to[2] = base64[((from[1] & 0xf) << 2) | (from[2] >> 6)]; + to[3] = base64[from[2] & 0x3f]; + + remaining -= 3; + to += 4; + from += 3; + } + + switch (remaining) { + case 2: + to[0] = base64[from[0] >> 2]; + to[1] = base64[((from[0] & 0x3) << 4) | (from[1] >> 4)]; + to[2] = base64[(from[1] & 0xf) << 2]; + to[3] = '='; + to += 4; + from += 2; + break; + case 1: + to[0] = base64[from[0] >> 2]; + to[1] = base64[((from[0] & 0x3) << 4)]; + to[2] = '='; + to[3] = '='; + to += 4; + from += 1; + break; + } + + bytes = to - data; + putstring(p, data, bytes); + print_data(p, "\"", 1); + return len; +} + +static void *scalar_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + CHK(putkey(closure, handler_data)); + print_data(p, "\"", 1); + return p; +} + +static size_t scalar_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool scalar_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\"", 1); + return true; +} + +static void *repeated_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_comma(p); + print_data(p, "\"", 1); + return p; +} + +static size_t repeated_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool repeated_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\"", 1); + return true; +} + +static void *mapkeyval_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_data(p, "\"", 1); + return p; +} + +static size_t mapkey_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool mapkey_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\":", 2); + return true; +} + +static bool mapvalue_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\"", 1); + return true; +} + +static size_t scalar_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putkey(closure, handler_data)); + CHK(putbytes(closure, handler_data, str, len, handle)); + return len; +} + +static size_t repeated_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + print_comma(p); + CHK(putbytes(closure, handler_data, str, len, handle)); + return len; +} + +static size_t mapkey_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + CHK(putbytes(closure, handler_data, str, len, handle)); + print_data(p, ":", 1); + return len; +} + +static void set_enum_hd(upb_handlers *h, + const upb_fielddef *f, + bool preserve_fieldnames, + upb_handlerattr *attr) { + EnumHandlerData *hd = upb_gmalloc(sizeof(EnumHandlerData)); + hd->enumdef = upb_fielddef_enumsubdef(f); + hd->keyname = newstrpc(h, f, preserve_fieldnames); + upb_handlers_addcleanup(h, hd, upb_gfree); + attr->handler_data = hd; +} + +/* Set up handlers for a mapentry submessage (i.e., an individual key/value pair + * in a map). + * + * TODO: Handle missing key, missing value, out-of-order key/value, or repeated + * key or value cases properly. The right way to do this is to allocate a + * temporary structure at the start of a mapentry submessage, store key and + * value data in it as key and value handlers are called, and then print the + * key/value pair once at the end of the submessage. If we don't do this, we + * should at least detect the case and throw an error. However, so far all of + * our sources that emit mapentry messages do so canonically (with one key + * field, and then one value field), so this is not a pressing concern at the + * moment. */ +void printer_sethandlers_mapentry(const void *closure, bool preserve_fieldnames, + upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + /* A mapentry message is printed simply as '"key": value'. Rather than + * special-case key and value for every type below, we just handle both + * fields explicitly here. */ + const upb_fielddef* key_field = upb_msgdef_itof(md, UPB_MAPENTRY_KEY); + const upb_fielddef* value_field = upb_msgdef_itof(md, UPB_MAPENTRY_VALUE); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + UPB_UNUSED(closure); + + switch (upb_fielddef_type(key_field)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, key_field, putmapkey_int32_t, &empty_attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, key_field, putmapkey_int64_t, &empty_attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, key_field, putmapkey_uint32_t, &empty_attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, key_field, putmapkey_uint64_t, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, key_field, putmapkey_bool, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, key_field, mapkeyval_startstr, &empty_attr); + upb_handlers_setstring(h, key_field, mapkey_str, &empty_attr); + upb_handlers_setendstr(h, key_field, mapkey_endstr, &empty_attr); + break; + case UPB_TYPE_BYTES: + upb_handlers_setstring(h, key_field, mapkey_bytes, &empty_attr); + break; + default: + UPB_ASSERT(false); + break; + } + + switch (upb_fielddef_type(value_field)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, value_field, putint32_t, &empty_attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, value_field, putint64_t, &empty_attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, value_field, putuint32_t, &empty_attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, value_field, putuint64_t, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, value_field, putbool, &empty_attr); + break; + case UPB_TYPE_FLOAT: + upb_handlers_setfloat(h, value_field, putfloat, &empty_attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, value_field, putdouble, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, value_field, mapkeyval_startstr, &empty_attr); + upb_handlers_setstring(h, value_field, putstr, &empty_attr); + upb_handlers_setendstr(h, value_field, mapvalue_endstr, &empty_attr); + break; + case UPB_TYPE_BYTES: + upb_handlers_setstring(h, value_field, putbytes, &empty_attr); + break; + case UPB_TYPE_ENUM: { + upb_handlerattr enum_attr = UPB_HANDLERATTR_INIT; + set_enum_hd(h, value_field, preserve_fieldnames, &enum_attr); + upb_handlers_setint32(h, value_field, mapvalue_enum, &enum_attr); + break; + } + case UPB_TYPE_MESSAGE: + /* No handler necessary -- the submsg handlers will print the message + * as appropriate. */ + break; + } +} + +static bool putseconds(void *closure, const void *handler_data, + int64_t seconds) { + upb_json_printer *p = closure; + p->seconds = seconds; + UPB_UNUSED(handler_data); + return true; +} + +static bool putnanos(void *closure, const void *handler_data, + int32_t nanos) { + upb_json_printer *p = closure; + p->nanos = nanos; + UPB_UNUSED(handler_data); + return true; +} + +static void *scalar_startstr_nokey(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_data(p, "\"", 1); + return p; +} + +static size_t putstr_nokey(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(handle); + print_data(p, "\"", 1); + putstring(p, str, len); + print_data(p, "\"", 1); + return len + 2; +} + +static void *startseq_nokey(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "[", 1); + return closure; +} + +static void *startseq_fieldmask(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_++; + p->first_elem_[p->depth_] = true; + return closure; +} + +static bool endseq_fieldmask(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_--; + return true; +} + +static void *repeated_startstr_fieldmask( + void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_comma(p); + return p; +} + +static size_t repeated_str_fieldmask( + void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + const char* limit = str + len; + bool upper = false; + size_t result_len = 0; + for (; str < limit; str++) { + if (*str == '_') { + upper = true; + continue; + } + if (upper && *str >= 'a' && *str <= 'z') { + char upper_char = toupper(*str); + CHK(putstr(closure, handler_data, &upper_char, 1, handle)); + } else { + CHK(putstr(closure, handler_data, str, 1, handle)); + } + upper = false; + result_len++; + } + return result_len; +} + +static void *startmap_nokey(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); + return closure; +} + +static bool putnull(void *closure, const void *handler_data, + int32_t null) { + upb_json_printer *p = closure; + print_data(p, "null", 4); + UPB_UNUSED(handler_data); + UPB_UNUSED(null); + return true; +} + +static bool printer_startdurationmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + return true; +} + +#define UPB_DURATION_MAX_JSON_LEN 23 +#define UPB_DURATION_MAX_NANO_LEN 9 + +static bool printer_enddurationmsg(void *closure, const void *handler_data, + upb_status *s) { + upb_json_printer *p = closure; + char buffer[UPB_DURATION_MAX_JSON_LEN]; + size_t base_len; + size_t curr; + size_t i; + + memset(buffer, 0, UPB_DURATION_MAX_JSON_LEN); + + if (p->seconds < -315576000000) { + upb_status_seterrf(s, "error parsing duration: " + "minimum acceptable value is " + "-315576000000"); + return false; + } + + if (p->seconds > 315576000000) { + upb_status_seterrf(s, "error serializing duration: " + "maximum acceptable value is " + "315576000000"); + return false; + } + + _upb_snprintf(buffer, sizeof(buffer), "%ld", (long)p->seconds); + base_len = strlen(buffer); + + if (p->nanos != 0) { + char nanos_buffer[UPB_DURATION_MAX_NANO_LEN + 3]; + _upb_snprintf(nanos_buffer, sizeof(nanos_buffer), "%.9f", + p->nanos / 1000000000.0); + /* Remove trailing 0. */ + for (i = UPB_DURATION_MAX_NANO_LEN + 2; + nanos_buffer[i] == '0'; i--) { + nanos_buffer[i] = 0; + } + strcpy(buffer + base_len, nanos_buffer + 1); + } + + curr = strlen(buffer); + strcpy(buffer + curr, "s"); + + p->seconds = 0; + p->nanos = 0; + + print_data(p, "\"", 1); + print_data(p, buffer, strlen(buffer)); + print_data(p, "\"", 1); + + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + + UPB_UNUSED(handler_data); + return true; +} + +static bool printer_starttimestampmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + return true; +} + +#define UPB_TIMESTAMP_MAX_JSON_LEN 31 +#define UPB_TIMESTAMP_BEFORE_NANO_LEN 19 +#define UPB_TIMESTAMP_MAX_NANO_LEN 9 + +static bool printer_endtimestampmsg(void *closure, const void *handler_data, + upb_status *s) { + upb_json_printer *p = closure; + char buffer[UPB_TIMESTAMP_MAX_JSON_LEN]; + time_t time = p->seconds; + size_t curr; + size_t i; + size_t year_length = + strftime(buffer, UPB_TIMESTAMP_MAX_JSON_LEN, "%Y", gmtime(&time)); + + if (p->seconds < -62135596800) { + upb_status_seterrf(s, "error parsing timestamp: " + "minimum acceptable value is " + "0001-01-01T00:00:00Z"); + return false; + } + + if (p->seconds > 253402300799) { + upb_status_seterrf(s, "error parsing timestamp: " + "maximum acceptable value is " + "9999-12-31T23:59:59Z"); + return false; + } + + /* strftime doesn't guarantee 4 digits for year. Prepend 0 by ourselves. */ + for (i = 0; i < 4 - year_length; i++) { + buffer[i] = '0'; + } + + strftime(buffer + (4 - year_length), UPB_TIMESTAMP_MAX_JSON_LEN, + "%Y-%m-%dT%H:%M:%S", gmtime(&time)); + if (p->nanos != 0) { + char nanos_buffer[UPB_TIMESTAMP_MAX_NANO_LEN + 3]; + _upb_snprintf(nanos_buffer, sizeof(nanos_buffer), "%.9f", + p->nanos / 1000000000.0); + /* Remove trailing 0. */ + for (i = UPB_TIMESTAMP_MAX_NANO_LEN + 2; + nanos_buffer[i] == '0'; i--) { + nanos_buffer[i] = 0; + } + strcpy(buffer + UPB_TIMESTAMP_BEFORE_NANO_LEN, nanos_buffer + 1); + } + + curr = strlen(buffer); + strcpy(buffer + curr, "Z"); + + p->seconds = 0; + p->nanos = 0; + + print_data(p, "\"", 1); + print_data(p, buffer, strlen(buffer)); + print_data(p, "\"", 1); + + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + return true; +} + +static bool printer_startmsg_noframe(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + return true; +} + +static bool printer_endmsg_noframe( + void *closure, const void *handler_data, upb_status *s) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +static bool printer_startmsg_fieldmask( + void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + print_data(p, "\"", 1); + return true; +} + +static bool printer_endmsg_fieldmask( + void *closure, const void *handler_data, upb_status *s) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + print_data(p, "\"", 1); + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +static void *scalar_startstr_onlykey( + void *closure, const void *handler_data, size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(size_hint); + CHK(putkey(closure, handler_data)); + return p; +} + +/* Set up handlers for an Any submessage. */ +void printer_sethandlers_any(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* type_field = upb_msgdef_itof(md, UPB_ANY_TYPE); + const upb_fielddef* value_field = upb_msgdef_itof(md, UPB_ANY_VALUE); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + /* type_url's json name is "@type" */ + upb_handlerattr type_name_attr = UPB_HANDLERATTR_INIT; + upb_handlerattr value_name_attr = UPB_HANDLERATTR_INIT; + strpc *type_url_json_name = newstrpc_str(h, "@type"); + strpc *value_json_name = newstrpc_str(h, "value"); + + type_name_attr.handler_data = type_url_json_name; + value_name_attr.handler_data = value_json_name; + + /* Set up handlers. */ + upb_handlers_setstartmsg(h, printer_startmsg, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg, &empty_attr); + + upb_handlers_setstartstr(h, type_field, scalar_startstr, &type_name_attr); + upb_handlers_setstring(h, type_field, scalar_str, &empty_attr); + upb_handlers_setendstr(h, type_field, scalar_endstr, &empty_attr); + + /* This is not the full and correct JSON encoding for the Any value field. It + * requires further processing by the wrapper code based on the type URL. + */ + upb_handlers_setstartstr(h, value_field, scalar_startstr_onlykey, + &value_name_attr); + + UPB_UNUSED(closure); +} + +/* Set up handlers for a fieldmask submessage. */ +void printer_sethandlers_fieldmask(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + const upb_fielddef* f = upb_msgdef_itof(md, 1); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartseq(h, f, startseq_fieldmask, &empty_attr); + upb_handlers_setendseq(h, f, endseq_fieldmask, &empty_attr); + + upb_handlers_setstartmsg(h, printer_startmsg_fieldmask, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_fieldmask, &empty_attr); + + upb_handlers_setstartstr(h, f, repeated_startstr_fieldmask, &empty_attr); + upb_handlers_setstring(h, f, repeated_str_fieldmask, &empty_attr); + + UPB_UNUSED(closure); +} + +/* Set up handlers for a duration submessage. */ +void printer_sethandlers_duration(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* seconds_field = + upb_msgdef_itof(md, UPB_DURATION_SECONDS); + const upb_fielddef* nanos_field = + upb_msgdef_itof(md, UPB_DURATION_NANOS); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartmsg(h, printer_startdurationmsg, &empty_attr); + upb_handlers_setint64(h, seconds_field, putseconds, &empty_attr); + upb_handlers_setint32(h, nanos_field, putnanos, &empty_attr); + upb_handlers_setendmsg(h, printer_enddurationmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +/* Set up handlers for a timestamp submessage. Instead of printing fields + * separately, the json representation of timestamp follows RFC 3339 */ +void printer_sethandlers_timestamp(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* seconds_field = + upb_msgdef_itof(md, UPB_TIMESTAMP_SECONDS); + const upb_fielddef* nanos_field = + upb_msgdef_itof(md, UPB_TIMESTAMP_NANOS); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartmsg(h, printer_starttimestampmsg, &empty_attr); + upb_handlers_setint64(h, seconds_field, putseconds, &empty_attr); + upb_handlers_setint32(h, nanos_field, putnanos, &empty_attr); + upb_handlers_setendmsg(h, printer_endtimestampmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +void printer_sethandlers_value(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + upb_msg_field_iter i; + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); + + upb_msg_field_begin(&i, md); + for(; !upb_msg_field_done(&i); upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_ENUM: + upb_handlers_setint32(h, f, putnull, &empty_attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, f, putdouble, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, f, scalar_startstr_nokey, &empty_attr); + upb_handlers_setstring(h, f, scalar_str, &empty_attr); + upb_handlers_setendstr(h, f, scalar_endstr, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, f, putbool, &empty_attr); + break; + case UPB_TYPE_MESSAGE: + break; + default: + UPB_ASSERT(false); + break; + } + } + + UPB_UNUSED(closure); +} + +#define WRAPPER_SETHANDLERS(wrapper, type, putmethod) \ +void printer_sethandlers_##wrapper(const void *closure, upb_handlers *h) { \ + const upb_msgdef *md = upb_handlers_msgdef(h); \ + const upb_fielddef* f = upb_msgdef_itof(md, 1); \ + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; \ + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); \ + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); \ + upb_handlers_set##type(h, f, putmethod, &empty_attr); \ + UPB_UNUSED(closure); \ +} + +WRAPPER_SETHANDLERS(doublevalue, double, putdouble) +WRAPPER_SETHANDLERS(floatvalue, float, putfloat) +WRAPPER_SETHANDLERS(int64value, int64, putint64_t) +WRAPPER_SETHANDLERS(uint64value, uint64, putuint64_t) +WRAPPER_SETHANDLERS(int32value, int32, putint32_t) +WRAPPER_SETHANDLERS(uint32value, uint32, putuint32_t) +WRAPPER_SETHANDLERS(boolvalue, bool, putbool) +WRAPPER_SETHANDLERS(stringvalue, string, putstr_nokey) +WRAPPER_SETHANDLERS(bytesvalue, string, putbytes) + +#undef WRAPPER_SETHANDLERS + +void printer_sethandlers_listvalue(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + const upb_fielddef* f = upb_msgdef_itof(md, 1); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartseq(h, f, startseq_nokey, &empty_attr); + upb_handlers_setendseq(h, f, endseq, &empty_attr); + + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); + + upb_handlers_setstartsubmsg(h, f, repeated_startsubmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +void printer_sethandlers_structvalue(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + const upb_fielddef* f = upb_msgdef_itof(md, 1); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartseq(h, f, startmap_nokey, &empty_attr); + upb_handlers_setendseq(h, f, endmap, &empty_attr); + + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); + + upb_handlers_setstartsubmsg(h, f, repeated_startsubmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +void printer_sethandlers(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + bool is_mapentry = upb_msgdef_mapentry(md); + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + upb_msg_field_iter i; + const upb_json_printercache *cache = closure; + const bool preserve_fieldnames = cache->preserve_fieldnames; + + if (is_mapentry) { + /* mapentry messages are sufficiently different that we handle them + * separately. */ + printer_sethandlers_mapentry(closure, preserve_fieldnames, h); + return; + } + + switch (upb_msgdef_wellknowntype(md)) { + case UPB_WELLKNOWN_UNSPECIFIED: + break; + case UPB_WELLKNOWN_ANY: + printer_sethandlers_any(closure, h); + return; + case UPB_WELLKNOWN_FIELDMASK: + printer_sethandlers_fieldmask(closure, h); + return; + case UPB_WELLKNOWN_DURATION: + printer_sethandlers_duration(closure, h); + return; + case UPB_WELLKNOWN_TIMESTAMP: + printer_sethandlers_timestamp(closure, h); + return; + case UPB_WELLKNOWN_VALUE: + printer_sethandlers_value(closure, h); + return; + case UPB_WELLKNOWN_LISTVALUE: + printer_sethandlers_listvalue(closure, h); + return; + case UPB_WELLKNOWN_STRUCT: + printer_sethandlers_structvalue(closure, h); + return; +#define WRAPPER(wellknowntype, name) \ + case wellknowntype: \ + printer_sethandlers_##name(closure, h); \ + return; \ + + WRAPPER(UPB_WELLKNOWN_DOUBLEVALUE, doublevalue); + WRAPPER(UPB_WELLKNOWN_FLOATVALUE, floatvalue); + WRAPPER(UPB_WELLKNOWN_INT64VALUE, int64value); + WRAPPER(UPB_WELLKNOWN_UINT64VALUE, uint64value); + WRAPPER(UPB_WELLKNOWN_INT32VALUE, int32value); + WRAPPER(UPB_WELLKNOWN_UINT32VALUE, uint32value); + WRAPPER(UPB_WELLKNOWN_BOOLVALUE, boolvalue); + WRAPPER(UPB_WELLKNOWN_STRINGVALUE, stringvalue); + WRAPPER(UPB_WELLKNOWN_BYTESVALUE, bytesvalue); + +#undef WRAPPER + } + + upb_handlers_setstartmsg(h, printer_startmsg, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg, &empty_attr); + +#define TYPE(type, name, ctype) \ + case type: \ + if (upb_fielddef_isseq(f)) { \ + upb_handlers_set##name(h, f, repeated_##ctype, &empty_attr); \ + } else { \ + upb_handlers_set##name(h, f, scalar_##ctype, &name_attr); \ + } \ + break; + + upb_msg_field_begin(&i, md); + for(; !upb_msg_field_done(&i); upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + + upb_handlerattr name_attr = UPB_HANDLERATTR_INIT; + name_attr.handler_data = newstrpc(h, f, preserve_fieldnames); + + if (upb_fielddef_ismap(f)) { + upb_handlers_setstartseq(h, f, startmap, &name_attr); + upb_handlers_setendseq(h, f, endmap, &name_attr); + } else if (upb_fielddef_isseq(f)) { + upb_handlers_setstartseq(h, f, startseq, &name_attr); + upb_handlers_setendseq(h, f, endseq, &empty_attr); + } + + switch (upb_fielddef_type(f)) { + TYPE(UPB_TYPE_FLOAT, float, float); + TYPE(UPB_TYPE_DOUBLE, double, double); + TYPE(UPB_TYPE_BOOL, bool, bool); + TYPE(UPB_TYPE_INT32, int32, int32_t); + TYPE(UPB_TYPE_UINT32, uint32, uint32_t); + TYPE(UPB_TYPE_INT64, int64, int64_t); + TYPE(UPB_TYPE_UINT64, uint64, uint64_t); + case UPB_TYPE_ENUM: { + /* For now, we always emit symbolic names for enums. We may want an + * option later to control this behavior, but we will wait for a real + * need first. */ + upb_handlerattr enum_attr = UPB_HANDLERATTR_INIT; + set_enum_hd(h, f, preserve_fieldnames, &enum_attr); + + if (upb_fielddef_isseq(f)) { + upb_handlers_setint32(h, f, repeated_enum, &enum_attr); + } else { + upb_handlers_setint32(h, f, scalar_enum, &enum_attr); + } + + break; + } + case UPB_TYPE_STRING: + if (upb_fielddef_isseq(f)) { + upb_handlers_setstartstr(h, f, repeated_startstr, &empty_attr); + upb_handlers_setstring(h, f, repeated_str, &empty_attr); + upb_handlers_setendstr(h, f, repeated_endstr, &empty_attr); + } else { + upb_handlers_setstartstr(h, f, scalar_startstr, &name_attr); + upb_handlers_setstring(h, f, scalar_str, &empty_attr); + upb_handlers_setendstr(h, f, scalar_endstr, &empty_attr); + } + break; + case UPB_TYPE_BYTES: + /* XXX: this doesn't support strings that span buffers yet. The base64 + * encoder will need to be made resumable for this to work properly. */ + if (upb_fielddef_isseq(f)) { + upb_handlers_setstring(h, f, repeated_bytes, &empty_attr); + } else { + upb_handlers_setstring(h, f, scalar_bytes, &name_attr); + } + break; + case UPB_TYPE_MESSAGE: + if (upb_fielddef_isseq(f)) { + upb_handlers_setstartsubmsg(h, f, repeated_startsubmsg, &name_attr); + } else { + upb_handlers_setstartsubmsg(h, f, scalar_startsubmsg, &name_attr); + } + break; + } + } + +#undef TYPE +} + +static void json_printer_reset(upb_json_printer *p) { + p->depth_ = 0; +} + + +/* Public API *****************************************************************/ + +upb_json_printer *upb_json_printer_create(upb_arena *a, const upb_handlers *h, + upb_bytessink output) { +#ifndef NDEBUG + size_t size_before = upb_arena_bytesallocated(a); +#endif + + upb_json_printer *p = upb_arena_malloc(a, sizeof(upb_json_printer)); + if (!p) return NULL; + + p->output_ = output; + json_printer_reset(p); + upb_sink_reset(&p->input_, h, p); + p->seconds = 0; + p->nanos = 0; + + /* If this fails, increase the value in printer.h. */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(a) - size_before <= + UPB_JSON_PRINTER_SIZE); + return p; +} + +upb_sink upb_json_printer_input(upb_json_printer *p) { + return p->input_; +} + +upb_handlercache *upb_json_printer_newcache(bool preserve_proto_fieldnames) { + upb_json_printercache *cache = upb_gmalloc(sizeof(*cache)); + upb_handlercache *ret = upb_handlercache_new(printer_sethandlers, cache); + + cache->preserve_fieldnames = preserve_proto_fieldnames; + upb_handlercache_addcleanup(ret, cache, upb_gfree); + + return ret; +} diff --git a/third_party/upb/upb/json/printer.h b/third_party/upb/upb/json/printer.h new file mode 100644 index 00000000000..85b9b128f97 --- /dev/null +++ b/third_party/upb/upb/json/printer.h @@ -0,0 +1,72 @@ +/* +** upb::json::Printer +** +** Handlers that emit JSON according to a specific protobuf schema. +*/ + +#ifndef UPB_JSON_TYPED_PRINTER_H_ +#define UPB_JSON_TYPED_PRINTER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace json { +class PrinterPtr; +} /* namespace json */ +} /* namespace upb */ +#endif + +/* upb_json_printer ***********************************************************/ + +#define UPB_JSON_PRINTER_SIZE 192 + +struct upb_json_printer; +typedef struct upb_json_printer upb_json_printer; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Native C API. */ +upb_json_printer *upb_json_printer_create(upb_arena *a, const upb_handlers *h, + upb_bytessink output); +upb_sink upb_json_printer_input(upb_json_printer *p); +const upb_handlers *upb_json_printer_newhandlers(const upb_msgdef *md, + bool preserve_fieldnames, + const void *owner); + +/* Lazily builds and caches handlers that will push encoded data to a bytessink. + * Any msgdef objects used with this object must outlive it. */ +upb_handlercache *upb_json_printer_newcache(bool preserve_proto_fieldnames); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Prints an incoming stream of data to a BytesSink in JSON format. */ +class upb::json::PrinterPtr { + public: + PrinterPtr(upb_json_printer* ptr) : ptr_(ptr) {} + + static PrinterPtr Create(Arena *arena, const upb::Handlers *handlers, + BytesSink output) { + return PrinterPtr( + upb_json_printer_create(arena->ptr(), handlers, output.sink())); + } + + /* The input to the printer. */ + Sink input() { return upb_json_printer_input(ptr_); } + + static const size_t kSize = UPB_JSON_PRINTER_SIZE; + + static HandlerCache NewCache(bool preserve_proto_fieldnames) { + return upb_json_printer_newcache(preserve_proto_fieldnames); + } + + private: + upb_json_printer* ptr_; +}; + +#endif /* __cplusplus */ + +#endif /* UPB_JSON_TYPED_PRINTER_H_ */ diff --git a/third_party/upb/upb/legacy_msg_reflection.c b/third_party/upb/upb/legacy_msg_reflection.c new file mode 100644 index 00000000000..031aa4e9cc7 --- /dev/null +++ b/third_party/upb/upb/legacy_msg_reflection.c @@ -0,0 +1,399 @@ + +#include "upb/legacy_msg_reflection.h" + +#include +#include "upb/table.int.h" +#include "upb/msg.h" + +#include "upb/port_def.inc" + +bool upb_fieldtype_mapkeyok(upb_fieldtype_t type) { + return type == UPB_TYPE_BOOL || type == UPB_TYPE_INT32 || + type == UPB_TYPE_UINT32 || type == UPB_TYPE_INT64 || + type == UPB_TYPE_UINT64 || type == UPB_TYPE_STRING; +} + +#define PTR_AT(msg, ofs, type) (type*)((char*)msg + ofs) +#define VOIDPTR_AT(msg, ofs) PTR_AT(msg, ofs, void) +#define ENCODE_MAX_NESTING 64 +#define CHECK_TRUE(x) if (!(x)) { return false; } + +/** upb_msgval ****************************************************************/ + +/* These functions will generate real memcpy() calls on ARM sadly, because + * the compiler assumes they might not be aligned. */ + +static upb_msgval upb_msgval_read(const void *p, size_t ofs, + uint8_t size) { + upb_msgval val; + p = (char*)p + ofs; + memcpy(&val, p, size); + return val; +} + +static void upb_msgval_write(void *p, size_t ofs, upb_msgval val, + uint8_t size) { + p = (char*)p + ofs; + memcpy(p, &val, size); +} + +static size_t upb_msgval_sizeof(upb_fieldtype_t type) { + switch (type) { + case UPB_TYPE_DOUBLE: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + return 8; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_FLOAT: + return 4; + case UPB_TYPE_BOOL: + return 1; + case UPB_TYPE_MESSAGE: + return sizeof(void*); + case UPB_TYPE_BYTES: + case UPB_TYPE_STRING: + return sizeof(upb_strview); + } + UPB_UNREACHABLE(); +} + +static uint8_t upb_msg_fieldsize(const upb_msglayout_field *field) { + if (field->label == UPB_LABEL_REPEATED) { + return sizeof(void*); + } else { + return upb_msgval_sizeof(upb_desctype_to_fieldtype[field->descriptortype]); + } +} + +/* TODO(haberman): this is broken right now because upb_msgval can contain + * a char* / size_t pair, which is too big for a upb_value. To fix this + * we'll probably need to dynamically allocate a upb_msgval and store a + * pointer to that in the tables for extensions/maps. */ +static upb_value upb_toval(upb_msgval val) { + upb_value ret; + UPB_UNUSED(val); + memset(&ret, 0, sizeof(upb_value)); /* XXX */ + return ret; +} + +static upb_msgval upb_msgval_fromval(upb_value val) { + upb_msgval ret; + UPB_UNUSED(val); + memset(&ret, 0, sizeof(upb_msgval)); /* XXX */ + return ret; +} + +static upb_ctype_t upb_fieldtotabtype(upb_fieldtype_t type) { + switch (type) { + case UPB_TYPE_FLOAT: return UPB_CTYPE_FLOAT; + case UPB_TYPE_DOUBLE: return UPB_CTYPE_DOUBLE; + case UPB_TYPE_BOOL: return UPB_CTYPE_BOOL; + case UPB_TYPE_BYTES: + case UPB_TYPE_MESSAGE: + case UPB_TYPE_STRING: return UPB_CTYPE_CONSTPTR; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: return UPB_CTYPE_INT32; + case UPB_TYPE_UINT32: return UPB_CTYPE_UINT32; + case UPB_TYPE_INT64: return UPB_CTYPE_INT64; + case UPB_TYPE_UINT64: return UPB_CTYPE_UINT64; + default: UPB_ASSERT(false); return 0; + } +} + + +/** upb_msg *******************************************************************/ + +/* If we always read/write as a consistent type to each address, this shouldn't + * violate aliasing. + */ +#define DEREF(msg, ofs, type) *PTR_AT(msg, ofs, type) + +static const upb_msglayout_field *upb_msg_checkfield(int field_index, + const upb_msglayout *l) { + UPB_ASSERT(field_index >= 0 && field_index < l->field_count); + return &l->fields[field_index]; +} + +static bool upb_msg_inoneof(const upb_msglayout_field *field) { + return field->presence < 0; +} + +static uint32_t *upb_msg_oneofcase(const upb_msg *msg, int field_index, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + UPB_ASSERT(upb_msg_inoneof(field)); + return PTR_AT(msg, ~field->presence, uint32_t); +} + +bool upb_msg_has(const upb_msg *msg, + int field_index, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + + UPB_ASSERT(field->presence); + + if (upb_msg_inoneof(field)) { + /* Oneofs are set when the oneof number is set to this field. */ + return *upb_msg_oneofcase(msg, field_index, l) == field->number; + } else { + /* Other fields are set when their hasbit is set. */ + uint32_t hasbit = field->presence; + return DEREF(msg, hasbit / 8, char) | (1 << (hasbit % 8)); + } +} + +upb_msgval upb_msg_get(const upb_msg *msg, int field_index, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + int size = upb_msg_fieldsize(field); + return upb_msgval_read(msg, field->offset, size); +} + +void upb_msg_set(upb_msg *msg, int field_index, upb_msgval val, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + int size = upb_msg_fieldsize(field); + upb_msgval_write(msg, field->offset, val, size); +} + + +/** upb_array *****************************************************************/ + +#define DEREF_ARR(arr, i, type) ((type*)arr->data)[i] + +size_t upb_array_size(const upb_array *arr) { + return arr->len; +} + +upb_msgval upb_array_get(const upb_array *arr, upb_fieldtype_t type, size_t i) { + size_t element_size = upb_msgval_sizeof(type); + UPB_ASSERT(i < arr->len); + return upb_msgval_read(arr->data, i * element_size, element_size); +} + +bool upb_array_set(upb_array *arr, upb_fieldtype_t type, size_t i, + upb_msgval val, upb_arena *arena) { + size_t element_size = upb_msgval_sizeof(type); + UPB_ASSERT(i <= arr->len); + + if (i == arr->len) { + /* Extending the array. */ + + if (i == arr->size) { + /* Need to reallocate. */ + size_t new_size = UPB_MAX(arr->size * 2, 8); + size_t new_bytes = new_size * element_size; + size_t old_bytes = arr->size * element_size; + upb_alloc *alloc = upb_arena_alloc(arena); + upb_msgval *new_data = + upb_realloc(alloc, arr->data, old_bytes, new_bytes); + + if (!new_data) { + return false; + } + + arr->data = new_data; + arr->size = new_size; + } + + arr->len = i + 1; + } + + upb_msgval_write(arr->data, i * element_size, val, element_size); + return true; +} + +/** upb_map *******************************************************************/ + +struct upb_map { + upb_fieldtype_t key_type; + upb_fieldtype_t val_type; + /* We may want to optimize this to use inttable where possible, for greater + * efficiency and lower memory footprint. */ + upb_strtable strtab; + upb_arena *arena; +}; + +static void upb_map_tokey(upb_fieldtype_t type, upb_msgval *key, + const char **out_key, size_t *out_len) { + switch (type) { + case UPB_TYPE_STRING: + /* Point to string data of the input key. */ + *out_key = key->str.data; + *out_len = key->str.size; + return; + case UPB_TYPE_BOOL: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + /* Point to the key itself. XXX: big-endian. */ + *out_key = (const char*)key; + *out_len = upb_msgval_sizeof(type); + return; + case UPB_TYPE_BYTES: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_ENUM: + case UPB_TYPE_FLOAT: + case UPB_TYPE_MESSAGE: + break; /* Cannot be a map key. */ + } + UPB_UNREACHABLE(); +} + +static upb_msgval upb_map_fromkey(upb_fieldtype_t type, const char *key, + size_t len) { + switch (type) { + case UPB_TYPE_STRING: + return upb_msgval_makestr(key, len); + case UPB_TYPE_BOOL: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + return upb_msgval_read(key, 0, upb_msgval_sizeof(type)); + case UPB_TYPE_BYTES: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_ENUM: + case UPB_TYPE_FLOAT: + case UPB_TYPE_MESSAGE: + break; /* Cannot be a map key. */ + } + UPB_UNREACHABLE(); +} + +upb_map *upb_map_new(upb_fieldtype_t ktype, upb_fieldtype_t vtype, + upb_arena *a) { + upb_ctype_t vtabtype = upb_fieldtotabtype(vtype); + upb_alloc *alloc = upb_arena_alloc(a); + upb_map *map = upb_malloc(alloc, sizeof(upb_map)); + + if (!map) { + return NULL; + } + + UPB_ASSERT(upb_fieldtype_mapkeyok(ktype)); + map->key_type = ktype; + map->val_type = vtype; + map->arena = a; + + if (!upb_strtable_init2(&map->strtab, vtabtype, alloc)) { + return NULL; + } + + return map; +} + +size_t upb_map_size(const upb_map *map) { + return upb_strtable_count(&map->strtab); +} + +upb_fieldtype_t upb_map_keytype(const upb_map *map) { + return map->key_type; +} + +upb_fieldtype_t upb_map_valuetype(const upb_map *map) { + return map->val_type; +} + +bool upb_map_get(const upb_map *map, upb_msgval key, upb_msgval *val) { + upb_value tabval; + const char *key_str; + size_t key_len; + bool ret; + + upb_map_tokey(map->key_type, &key, &key_str, &key_len); + ret = upb_strtable_lookup2(&map->strtab, key_str, key_len, &tabval); + if (ret) { + memcpy(val, &tabval, sizeof(tabval)); + } + + return ret; +} + +bool upb_map_set(upb_map *map, upb_msgval key, upb_msgval val, + upb_msgval *removed) { + const char *key_str; + size_t key_len; + upb_value tabval = upb_toval(val); + upb_value removedtabval; + upb_alloc *a = upb_arena_alloc(map->arena); + + upb_map_tokey(map->key_type, &key, &key_str, &key_len); + + /* TODO(haberman): add overwrite operation to minimize number of lookups. */ + if (upb_strtable_lookup2(&map->strtab, key_str, key_len, NULL)) { + upb_strtable_remove3(&map->strtab, key_str, key_len, &removedtabval, a); + memcpy(&removed, &removedtabval, sizeof(removed)); + } + + return upb_strtable_insert3(&map->strtab, key_str, key_len, tabval, a); +} + +bool upb_map_del(upb_map *map, upb_msgval key) { + const char *key_str; + size_t key_len; + upb_alloc *a = upb_arena_alloc(map->arena); + + upb_map_tokey(map->key_type, &key, &key_str, &key_len); + return upb_strtable_remove3(&map->strtab, key_str, key_len, NULL, a); +} + + +/** upb_mapiter ***************************************************************/ + +struct upb_mapiter { + upb_strtable_iter iter; + upb_fieldtype_t key_type; +}; + +size_t upb_mapiter_sizeof(void) { + return sizeof(upb_mapiter); +} + +void upb_mapiter_begin(upb_mapiter *i, const upb_map *map) { + upb_strtable_begin(&i->iter, &map->strtab); + i->key_type = map->key_type; +} + +upb_mapiter *upb_mapiter_new(const upb_map *t, upb_alloc *a) { + upb_mapiter *ret = upb_malloc(a, upb_mapiter_sizeof()); + + if (!ret) { + return NULL; + } + + upb_mapiter_begin(ret, t); + return ret; +} + +void upb_mapiter_free(upb_mapiter *i, upb_alloc *a) { + upb_free(a, i); +} + +void upb_mapiter_next(upb_mapiter *i) { + upb_strtable_next(&i->iter); +} + +bool upb_mapiter_done(const upb_mapiter *i) { + return upb_strtable_done(&i->iter); +} + +upb_msgval upb_mapiter_key(const upb_mapiter *i) { + return upb_map_fromkey(i->key_type, upb_strtable_iter_key(&i->iter), + upb_strtable_iter_keylength(&i->iter)); +} + +upb_msgval upb_mapiter_value(const upb_mapiter *i) { + return upb_msgval_fromval(upb_strtable_iter_value(&i->iter)); +} + +void upb_mapiter_setdone(upb_mapiter *i) { + upb_strtable_iter_setdone(&i->iter); +} + +bool upb_mapiter_isequal(const upb_mapiter *i1, const upb_mapiter *i2) { + return upb_strtable_iter_isequal(&i1->iter, &i2->iter); +} diff --git a/third_party/upb/upb/legacy_msg_reflection.h b/third_party/upb/upb/legacy_msg_reflection.h new file mode 100644 index 00000000000..c54bfb947a8 --- /dev/null +++ b/third_party/upb/upb/legacy_msg_reflection.h @@ -0,0 +1,191 @@ + +#ifndef UPB_LEGACY_MSG_REFLECTION_H_ +#define UPB_LEGACY_MSG_REFLECTION_H_ + +#include "upb/upb.h" +#include "upb/msg.h" + +#include "upb/port_def.inc" + +struct upb_map; +typedef struct upb_map upb_map; + +struct upb_mapiter; +typedef struct upb_mapiter upb_mapiter; + +/** upb_msgval ****************************************************************/ + +/* A union representing all possible protobuf values. Used for generic get/set + * operations. */ + +typedef union { + bool b; + float flt; + double dbl; + int32_t i32; + int64_t i64; + uint32_t u32; + uint64_t u64; + const upb_map* map; + const upb_msg* msg; + const upb_array* arr; + const void* ptr; + upb_strview str; +} upb_msgval; + +#define ACCESSORS(name, membername, ctype) \ + UPB_INLINE ctype upb_msgval_get ## name(upb_msgval v) { \ + return v.membername; \ + } \ + UPB_INLINE void upb_msgval_set ## name(upb_msgval *v, ctype cval) { \ + v->membername = cval; \ + } \ + UPB_INLINE upb_msgval upb_msgval_ ## name(ctype v) { \ + upb_msgval ret; \ + ret.membername = v; \ + return ret; \ + } + +ACCESSORS(bool, b, bool) +ACCESSORS(float, flt, float) +ACCESSORS(double, dbl, double) +ACCESSORS(int32, i32, int32_t) +ACCESSORS(int64, i64, int64_t) +ACCESSORS(uint32, u32, uint32_t) +ACCESSORS(uint64, u64, uint64_t) +ACCESSORS(map, map, const upb_map*) +ACCESSORS(msg, msg, const upb_msg*) +ACCESSORS(ptr, ptr, const void*) +ACCESSORS(arr, arr, const upb_array*) +ACCESSORS(str, str, upb_strview) + +#undef ACCESSORS + +UPB_INLINE upb_msgval upb_msgval_makestr(const char *data, size_t size) { + return upb_msgval_str(upb_strview_make(data, size)); +} + +/** upb_msg *******************************************************************/ + +/* A upb_msg represents a protobuf message. It always corresponds to a specific + * upb_msglayout, which describes how it is laid out in memory. */ + +/* Read-only message API. Can be safely called by anyone. */ + +/* Returns the value associated with this field: + * - for scalar fields (including strings), the value directly. + * - return upb_msg*, or upb_map* for msg/map. + * If the field is unset for these field types, returns NULL. + * + * TODO(haberman): should we let users store cached array/map/msg + * pointers here for fields that are unset? Could be useful for the + * strongly-owned submessage model (ie. generated C API that doesn't use + * arenas). + */ +upb_msgval upb_msg_get(const upb_msg *msg, + int field_index, + const upb_msglayout *l); + +/* May only be called for fields where upb_fielddef_haspresence(f) == true. */ +bool upb_msg_has(const upb_msg *msg, + int field_index, + const upb_msglayout *l); + +/* Mutable message API. May only be called by the owner of the message who + * knows its ownership scheme and how to keep it consistent. */ + +/* Sets the given field to the given value. Does not perform any memory + * management: if you overwrite a pointer to a msg/array/map/string without + * cleaning it up (or using an arena) it will leak. + */ +void upb_msg_set(upb_msg *msg, + int field_index, + upb_msgval val, + const upb_msglayout *l); + +/* For a primitive field, set it back to its default. For repeated, string, and + * submessage fields set it back to NULL. This could involve releasing some + * internal memory (for example, from an extension dictionary), but it is not + * recursive in any way and will not recover any memory that may be used by + * arrays/maps/strings/msgs that this field may have pointed to. + */ +bool upb_msg_clearfield(upb_msg *msg, + int field_index, + const upb_msglayout *l); + +/* TODO(haberman): copyfrom()/mergefrom()? */ + +/** upb_array *****************************************************************/ + +/* A upb_array stores data for a repeated field. The memory management + * semantics are the same as upb_msg. A upb_array allocates dynamic + * memory internally for the array elements. */ + +upb_fieldtype_t upb_array_type(const upb_array *arr); + +/* Read-only interface. Safe for anyone to call. */ + +size_t upb_array_size(const upb_array *arr); +upb_msgval upb_array_get(const upb_array *arr, upb_fieldtype_t type, size_t i); + +/* Write interface. May only be called by the message's owner who can enforce + * its memory management invariants. */ + +bool upb_array_set(upb_array *arr, upb_fieldtype_t type, size_t i, + upb_msgval val, upb_arena *arena); + +/** upb_map *******************************************************************/ + +/* A upb_map stores data for a map field. The memory management semantics are + * the same as upb_msg, with one notable exception. upb_map will internally + * store a copy of all string keys, but *not* any string values or submessages. + * So you must ensure that any string or message values outlive the map, and you + * must delete them manually when they are no longer required. */ + +upb_map *upb_map_new(upb_fieldtype_t ktype, upb_fieldtype_t vtype, + upb_arena *a); + +/* Read-only interface. Safe for anyone to call. */ + +size_t upb_map_size(const upb_map *map); +upb_fieldtype_t upb_map_keytype(const upb_map *map); +upb_fieldtype_t upb_map_valuetype(const upb_map *map); +bool upb_map_get(const upb_map *map, upb_msgval key, upb_msgval *val); + +/* Write interface. May only be called by the message's owner who can enforce + * its memory management invariants. */ + +/* Sets or overwrites an entry in the map. Return value indicates whether + * the operation succeeded or failed with OOM, and also whether an existing + * key was replaced or not. */ +bool upb_map_set(upb_map *map, + upb_msgval key, upb_msgval val, + upb_msgval *valremoved); + +/* Deletes an entry in the map. Returns true if the key was present. */ +bool upb_map_del(upb_map *map, upb_msgval key); + +/** upb_mapiter ***************************************************************/ + +/* For iterating over a map. Map iterators are invalidated by mutations to the + * map, but an invalidated iterator will never return junk or crash the process. + * An invalidated iterator may return entries that were already returned though, + * and if you keep invalidating the iterator during iteration, the program may + * enter an infinite loop. */ + +size_t upb_mapiter_sizeof(void); + +void upb_mapiter_begin(upb_mapiter *i, const upb_map *t); +upb_mapiter *upb_mapiter_new(const upb_map *t, upb_alloc *a); +void upb_mapiter_free(upb_mapiter *i, upb_alloc *a); +void upb_mapiter_next(upb_mapiter *i); +bool upb_mapiter_done(const upb_mapiter *i); + +upb_msgval upb_mapiter_key(const upb_mapiter *i); +upb_msgval upb_mapiter_value(const upb_mapiter *i); +void upb_mapiter_setdone(upb_mapiter *i); +bool upb_mapiter_isequal(const upb_mapiter *i1, const upb_mapiter *i2); + +#include "upb/port_undef.inc" + +#endif /* UPB_LEGACY_MSG_REFLECTION_H_ */ diff --git a/third_party/upb/upb/msg.c b/third_party/upb/upb/msg.c new file mode 100644 index 00000000000..a77da5665c7 --- /dev/null +++ b/third_party/upb/upb/msg.c @@ -0,0 +1,111 @@ + +#include "upb/msg.h" + +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +#define VOIDPTR_AT(msg, ofs) (void*)((char*)msg + (int)ofs) + +/* Internal members of a upb_msg. We can change this without breaking binary + * compatibility. We put these before the user's data. The user's upb_msg* + * points after the upb_msg_internal. */ + +/* Used when a message is not extendable. */ +typedef struct { + char *unknown; + size_t unknown_len; + size_t unknown_size; +} upb_msg_internal; + +/* Used when a message is extendable. */ +typedef struct { + upb_inttable *extdict; + upb_msg_internal base; +} upb_msg_internal_withext; + +static int upb_msg_internalsize(const upb_msglayout *l) { + return sizeof(upb_msg_internal) - l->extendable * sizeof(void *); +} + +static size_t upb_msg_sizeof(const upb_msglayout *l) { + return l->size + upb_msg_internalsize(l); +} + +static upb_msg_internal *upb_msg_getinternal(upb_msg *msg) { + return VOIDPTR_AT(msg, -sizeof(upb_msg_internal)); +} + +static const upb_msg_internal *upb_msg_getinternal_const(const upb_msg *msg) { + return VOIDPTR_AT(msg, -sizeof(upb_msg_internal)); +} + +static upb_msg_internal_withext *upb_msg_getinternalwithext( + upb_msg *msg, const upb_msglayout *l) { + UPB_ASSERT(l->extendable); + return VOIDPTR_AT(msg, -sizeof(upb_msg_internal_withext)); +} + +upb_msg *upb_msg_new(const upb_msglayout *l, upb_arena *a) { + upb_alloc *alloc = upb_arena_alloc(a); + void *mem = upb_malloc(alloc, upb_msg_sizeof(l)); + upb_msg_internal *in; + upb_msg *msg; + + if (!mem) { + return NULL; + } + + msg = VOIDPTR_AT(mem, upb_msg_internalsize(l)); + + /* Initialize normal members. */ + memset(msg, 0, l->size); + + /* Initialize internal members. */ + in = upb_msg_getinternal(msg); + in->unknown = NULL; + in->unknown_len = 0; + in->unknown_size = 0; + + if (l->extendable) { + upb_msg_getinternalwithext(msg, l)->extdict = NULL; + } + + return msg; +} + +upb_array *upb_array_new(upb_arena *a) { + upb_array *ret = upb_arena_malloc(a, sizeof(upb_array)); + + if (!ret) { + return NULL; + } + + ret->data = NULL; + ret->len = 0; + ret->size = 0; + + return ret; +} + +void upb_msg_addunknown(upb_msg *msg, const char *data, size_t len, + upb_arena *arena) { + upb_msg_internal *in = upb_msg_getinternal(msg); + if (len > in->unknown_size - in->unknown_len) { + upb_alloc *alloc = upb_arena_alloc(arena); + size_t need = in->unknown_size + len; + size_t newsize = UPB_MAX(in->unknown_size * 2, need); + in->unknown = upb_realloc(alloc, in->unknown, in->unknown_size, newsize); + in->unknown_size = newsize; + } + memcpy(in->unknown + in->unknown_len, data, len); + in->unknown_len += len; +} + +const char *upb_msg_getunknown(const upb_msg *msg, size_t *len) { + const upb_msg_internal* in = upb_msg_getinternal_const(msg); + *len = in->unknown_len; + return in->unknown; +} + +#undef VOIDPTR_AT diff --git a/third_party/upb/upb/msg.h b/third_party/upb/upb/msg.h new file mode 100644 index 00000000000..4bec023bd3e --- /dev/null +++ b/third_party/upb/upb/msg.h @@ -0,0 +1,69 @@ +/* +** Data structures for message tables, used for parsing and serialization. +** This are much lighter-weight than full reflection, but they are do not +** have enough information to convert to text format, JSON, etc. +** +** The definitions in this file are internal to upb. +**/ + +#ifndef UPB_MSG_H_ +#define UPB_MSG_H_ + +#include +#include +#include "upb/upb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void upb_msg; + +/** upb_msglayout *************************************************************/ + +/* upb_msglayout represents the memory layout of a given upb_msgdef. The + * members are public so generated code can initialize them, but users MUST NOT + * read or write any of its members. */ + +typedef struct { + uint32_t number; + uint16_t offset; + int16_t presence; /* If >0, hasbit_index+1. If <0, oneof_index+1. */ + uint16_t submsg_index; /* undefined if descriptortype != MESSAGE or GROUP. */ + uint8_t descriptortype; + uint8_t label; +} upb_msglayout_field; + +typedef struct upb_msglayout { + const struct upb_msglayout *const* submsgs; + const upb_msglayout_field *fields; + /* Must be aligned to sizeof(void*). Doesn't include internal members like + * unknown fields, extension dict, pointer to msglayout, etc. */ + uint16_t size; + uint16_t field_count; + bool extendable; +} upb_msglayout; + +/** Message internal representation *******************************************/ + +/* Our internal representation for repeated fields. */ +typedef struct { + void *data; /* Each element is element_size. */ + size_t len; /* Measured in elements. */ + size_t size; /* Measured in elements. */ +} upb_array; + +upb_msg *upb_msg_new(const upb_msglayout *l, upb_arena *a); +upb_msg *upb_msg_new(const upb_msglayout *l, upb_arena *a); + +void upb_msg_addunknown(upb_msg *msg, const char *data, size_t len, + upb_arena *arena); +const char *upb_msg_getunknown(const upb_msg *msg, size_t *len); + +upb_array *upb_array_new(upb_arena *a); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_MSG_H_ */ diff --git a/third_party/upb/upb/msgfactory.c b/third_party/upb/upb/msgfactory.c new file mode 100644 index 00000000000..4ecf2725b15 --- /dev/null +++ b/third_party/upb/upb/msgfactory.c @@ -0,0 +1,248 @@ + +#include "upb/msgfactory.h" + +#include "upb/port_def.inc" + +static bool is_power_of_two(size_t val) { + return (val & (val - 1)) == 0; +} + +/* Align up to the given power of 2. */ +static size_t align_up(size_t val, size_t align) { + UPB_ASSERT(is_power_of_two(align)); + return (val + align - 1) & ~(align - 1); +} + +static size_t div_round_up(size_t n, size_t d) { + return (n + d - 1) / d; +} + +static size_t upb_msgval_sizeof2(upb_fieldtype_t type) { + switch (type) { + case UPB_TYPE_DOUBLE: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + return 8; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_FLOAT: + return 4; + case UPB_TYPE_BOOL: + return 1; + case UPB_TYPE_MESSAGE: + return sizeof(void*); + case UPB_TYPE_BYTES: + case UPB_TYPE_STRING: + return sizeof(upb_strview); + } + UPB_UNREACHABLE(); +} + +static uint8_t upb_msg_fielddefsize(const upb_fielddef *f) { + if (upb_fielddef_isseq(f)) { + return sizeof(void*); + } else { + return upb_msgval_sizeof2(upb_fielddef_type(f)); + } +} + + +/** upb_msglayout *************************************************************/ + +static void upb_msglayout_free(upb_msglayout *l) { + upb_gfree(l); +} + +static size_t upb_msglayout_place(upb_msglayout *l, size_t size) { + size_t ret; + + l->size = align_up(l->size, size); + ret = l->size; + l->size += size; + return ret; +} + +static bool upb_msglayout_init(const upb_msgdef *m, + upb_msglayout *l, + upb_msgfactory *factory) { + upb_msg_field_iter it; + upb_msg_oneof_iter oit; + size_t hasbit; + size_t submsg_count = 0; + const upb_msglayout **submsgs; + upb_msglayout_field *fields; + + for (upb_msg_field_begin(&it, m); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + const upb_fielddef* f = upb_msg_iter_field(&it); + if (upb_fielddef_issubmsg(f)) { + submsg_count++; + } + } + + memset(l, 0, sizeof(*l)); + + fields = upb_gmalloc(upb_msgdef_numfields(m) * sizeof(*fields)); + submsgs = upb_gmalloc(submsg_count * sizeof(*submsgs)); + + if ((!fields && upb_msgdef_numfields(m)) || + (!submsgs && submsg_count)) { + /* OOM. */ + upb_gfree(fields); + upb_gfree(submsgs); + return false; + } + + l->field_count = upb_msgdef_numfields(m); + l->fields = fields; + l->submsgs = submsgs; + + /* Allocate data offsets in three stages: + * + * 1. hasbits. + * 2. regular fields. + * 3. oneof fields. + * + * OPT: There is a lot of room for optimization here to minimize the size. + */ + + /* Allocate hasbits and set basic field attributes. */ + submsg_count = 0; + for (upb_msg_field_begin(&it, m), hasbit = 0; + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + const upb_fielddef* f = upb_msg_iter_field(&it); + upb_msglayout_field *field = &fields[upb_fielddef_index(f)]; + + field->number = upb_fielddef_number(f); + field->descriptortype = upb_fielddef_descriptortype(f); + field->label = upb_fielddef_label(f); + + if (upb_fielddef_issubmsg(f)) { + const upb_msglayout *sub_layout = + upb_msgfactory_getlayout(factory, upb_fielddef_msgsubdef(f)); + field->submsg_index = submsg_count++; + submsgs[field->submsg_index] = sub_layout; + } + + if (upb_fielddef_haspresence(f) && !upb_fielddef_containingoneof(f)) { + field->presence = (hasbit++); + } else { + field->presence = 0; + } + } + + /* Account for space used by hasbits. */ + l->size = div_round_up(hasbit, 8); + + /* Allocate non-oneof fields. */ + for (upb_msg_field_begin(&it, m); !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + const upb_fielddef* f = upb_msg_iter_field(&it); + size_t field_size = upb_msg_fielddefsize(f); + size_t index = upb_fielddef_index(f); + + if (upb_fielddef_containingoneof(f)) { + /* Oneofs are handled separately below. */ + continue; + } + + fields[index].offset = upb_msglayout_place(l, field_size); + } + + /* Allocate oneof fields. Each oneof field consists of a uint32 for the case + * and space for the actual data. */ + for (upb_msg_oneof_begin(&oit, m); !upb_msg_oneof_done(&oit); + upb_msg_oneof_next(&oit)) { + const upb_oneofdef* o = upb_msg_iter_oneof(&oit); + upb_oneof_iter fit; + + size_t case_size = sizeof(uint32_t); /* Could potentially optimize this. */ + size_t field_size = 0; + uint32_t case_offset; + uint32_t data_offset; + + /* Calculate field size: the max of all field sizes. */ + for (upb_oneof_begin(&fit, o); + !upb_oneof_done(&fit); + upb_oneof_next(&fit)) { + const upb_fielddef* f = upb_oneof_iter_field(&fit); + field_size = UPB_MAX(field_size, upb_msg_fielddefsize(f)); + } + + /* Align and allocate case offset. */ + case_offset = upb_msglayout_place(l, case_size); + data_offset = upb_msglayout_place(l, field_size); + + for (upb_oneof_begin(&fit, o); + !upb_oneof_done(&fit); + upb_oneof_next(&fit)) { + const upb_fielddef* f = upb_oneof_iter_field(&fit); + fields[upb_fielddef_index(f)].offset = data_offset; + fields[upb_fielddef_index(f)].presence = ~case_offset; + } + } + + /* Size of the entire structure should be a multiple of its greatest + * alignment. TODO: track overall alignment for real? */ + l->size = align_up(l->size, 8); + + return true; +} + + +/** upb_msgfactory ************************************************************/ + +struct upb_msgfactory { + const upb_symtab *symtab; /* We own a ref. */ + upb_inttable layouts; +}; + +upb_msgfactory *upb_msgfactory_new(const upb_symtab *symtab) { + upb_msgfactory *ret = upb_gmalloc(sizeof(*ret)); + + ret->symtab = symtab; + upb_inttable_init(&ret->layouts, UPB_CTYPE_PTR); + + return ret; +} + +void upb_msgfactory_free(upb_msgfactory *f) { + upb_inttable_iter i; + upb_inttable_begin(&i, &f->layouts); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_msglayout *l = upb_value_getptr(upb_inttable_iter_value(&i)); + upb_msglayout_free(l); + } + + upb_inttable_uninit(&f->layouts); + upb_gfree(f); +} + +const upb_symtab *upb_msgfactory_symtab(const upb_msgfactory *f) { + return f->symtab; +} + +const upb_msglayout *upb_msgfactory_getlayout(upb_msgfactory *f, + const upb_msgdef *m) { + upb_value v; + UPB_ASSERT(upb_symtab_lookupmsg(f->symtab, upb_msgdef_fullname(m)) == m); + UPB_ASSERT(!upb_msgdef_mapentry(m)); + + if (upb_inttable_lookupptr(&f->layouts, m, &v)) { + UPB_ASSERT(upb_value_getptr(v)); + return upb_value_getptr(v); + } else { + /* In case of circular dependency, layout has to be inserted first. */ + upb_msglayout *l = upb_gmalloc(sizeof(*l)); + upb_msgfactory *mutable_f = (void*)f; + upb_inttable_insertptr(&mutable_f->layouts, m, upb_value_ptr(l)); + UPB_ASSERT(l); + if (!upb_msglayout_init(m, l, f)) { + upb_msglayout_free(l); + } + return l; + } +} diff --git a/third_party/upb/upb/msgfactory.h b/third_party/upb/upb/msgfactory.h new file mode 100644 index 00000000000..9b3b5999389 --- /dev/null +++ b/third_party/upb/upb/msgfactory.h @@ -0,0 +1,48 @@ + +#include "upb/def.h" +#include "upb/msg.h" + +#ifndef UPB_MSGFACTORY_H_ +#define UPB_MSGFACTORY_H_ + +/** upb_msgfactory ************************************************************/ + +struct upb_msgfactory; +typedef struct upb_msgfactory upb_msgfactory; + +#ifdef __cplusplus +extern "C" { +#endif + +/* A upb_msgfactory contains a cache of upb_msglayout, upb_handlers, and + * upb_visitorplan objects. These are the objects necessary to represent, + * populate, and and visit upb_msg objects. + * + * These caches are all populated by upb_msgdef, and lazily created on demand. + */ + +/* Creates and destroys a msgfactory, respectively. The messages for this + * msgfactory must come from |symtab| (which should outlive the msgfactory). */ +upb_msgfactory *upb_msgfactory_new(const upb_symtab *symtab); +void upb_msgfactory_free(upb_msgfactory *f); + +const upb_symtab *upb_msgfactory_symtab(const upb_msgfactory *f); + +/* The functions to get cached objects, lazily creating them on demand. These + * all require: + * + * - m is in upb_msgfactory_symtab(f) + * - upb_msgdef_mapentry(m) == false (since map messages can't have layouts). + * + * The returned objects will live for as long as the msgfactory does. + * + * TODO(haberman): consider making this thread-safe and take a const + * upb_msgfactory. */ +const upb_msglayout *upb_msgfactory_getlayout(upb_msgfactory *f, + const upb_msgdef *m); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_MSGFACTORY_H_ */ diff --git a/third_party/upb/upb/pb/compile_decoder.c b/third_party/upb/upb/pb/compile_decoder.c new file mode 100644 index 00000000000..f5c7d656466 --- /dev/null +++ b/third_party/upb/upb/pb/compile_decoder.c @@ -0,0 +1,919 @@ +/* +** protobuf decoder bytecode compiler +** +** Code to compile a upb::Handlers into bytecode for decoding a protobuf +** according to that specific schema and destination handlers. +** +** Bytecode definition is in decoder.int.h. +*/ + +#include +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" + +#ifdef UPB_DUMP_BYTECODE +#include +#endif + +#include "upb/port_def.inc" + +#define MAXLABEL 5 +#define EMPTYLABEL -1 + +/* upb_pbdecodermethod ********************************************************/ + +static void freemethod(upb_pbdecodermethod *method) { + upb_inttable_uninit(&method->dispatch); + upb_gfree(method); +} + +static upb_pbdecodermethod *newmethod(const upb_handlers *dest_handlers, + mgroup *group) { + upb_pbdecodermethod *ret = upb_gmalloc(sizeof(*ret)); + upb_byteshandler_init(&ret->input_handler_); + + ret->group = group; + ret->dest_handlers_ = dest_handlers; + upb_inttable_init(&ret->dispatch, UPB_CTYPE_UINT64); + + return ret; +} + +const upb_handlers *upb_pbdecodermethod_desthandlers( + const upb_pbdecodermethod *m) { + return m->dest_handlers_; +} + +const upb_byteshandler *upb_pbdecodermethod_inputhandler( + const upb_pbdecodermethod *m) { + return &m->input_handler_; +} + +bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m) { + return m->is_native_; +} + + +/* mgroup *********************************************************************/ + +static void freegroup(mgroup *g) { + upb_inttable_iter i; + + upb_inttable_begin(&i, &g->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + freemethod(upb_value_getptr(upb_inttable_iter_value(&i))); + } + + upb_inttable_uninit(&g->methods); + upb_gfree(g->bytecode); + upb_gfree(g); +} + +mgroup *newgroup(void) { + mgroup *g = upb_gmalloc(sizeof(*g)); + upb_inttable_init(&g->methods, UPB_CTYPE_PTR); + g->bytecode = NULL; + g->bytecode_end = NULL; + return g; +} + + +/* bytecode compiler **********************************************************/ + +/* Data used only at compilation time. */ +typedef struct { + mgroup *group; + + uint32_t *pc; + int fwd_labels[MAXLABEL]; + int back_labels[MAXLABEL]; + + /* For fields marked "lazy", parse them lazily or eagerly? */ + bool lazy; +} compiler; + +static compiler *newcompiler(mgroup *group, bool lazy) { + compiler *ret = upb_gmalloc(sizeof(*ret)); + int i; + + ret->group = group; + ret->lazy = lazy; + for (i = 0; i < MAXLABEL; i++) { + ret->fwd_labels[i] = EMPTYLABEL; + ret->back_labels[i] = EMPTYLABEL; + } + return ret; +} + +static void freecompiler(compiler *c) { + upb_gfree(c); +} + +const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); + +/* How many words an instruction is. */ +static int instruction_len(uint32_t instr) { + switch (getop(instr)) { + case OP_SETDISPATCH: return 1 + ptr_words; + case OP_TAGN: return 3; + case OP_SETBIGGROUPNUM: return 2; + default: return 1; + } +} + +bool op_has_longofs(int32_t instruction) { + switch (getop(instruction)) { + case OP_CALL: + case OP_BRANCH: + case OP_CHECKDELIM: + return true; + /* The "tag" instructions only have 8 bytes available for the jump target, + * but that is ok because these opcodes only require short jumps. */ + case OP_TAG1: + case OP_TAG2: + case OP_TAGN: + return false; + default: + UPB_ASSERT(false); + return false; + } +} + +static int32_t getofs(uint32_t instruction) { + if (op_has_longofs(instruction)) { + return (int32_t)instruction >> 8; + } else { + return (int8_t)(instruction >> 8); + } +} + +static void setofs(uint32_t *instruction, int32_t ofs) { + if (op_has_longofs(*instruction)) { + *instruction = getop(*instruction) | (uint32_t)ofs << 8; + } else { + *instruction = (*instruction & ~0xff00) | ((ofs & 0xff) << 8); + } + UPB_ASSERT(getofs(*instruction) == ofs); /* Would fail in cases of overflow. */ +} + +static uint32_t pcofs(compiler *c) { return c->pc - c->group->bytecode; } + +/* Defines a local label at the current PC location. All previous forward + * references are updated to point to this location. The location is noted + * for any future backward references. */ +static void label(compiler *c, unsigned int label) { + int val; + uint32_t *codep; + + UPB_ASSERT(label < MAXLABEL); + val = c->fwd_labels[label]; + codep = (val == EMPTYLABEL) ? NULL : c->group->bytecode + val; + while (codep) { + int ofs = getofs(*codep); + setofs(codep, c->pc - codep - instruction_len(*codep)); + codep = ofs ? codep + ofs : NULL; + } + c->fwd_labels[label] = EMPTYLABEL; + c->back_labels[label] = pcofs(c); +} + +/* Creates a reference to a numbered label; either a forward reference + * (positive arg) or backward reference (negative arg). For forward references + * the value returned now is actually a "next" pointer into a linked list of all + * instructions that use this label and will be patched later when the label is + * defined with label(). + * + * The returned value is the offset that should be written into the instruction. + */ +static int32_t labelref(compiler *c, int label) { + UPB_ASSERT(label < MAXLABEL); + if (label == LABEL_DISPATCH) { + /* No resolving required. */ + return 0; + } else if (label < 0) { + /* Backward local label. Relative to the next instruction. */ + uint32_t from = (c->pc + 1) - c->group->bytecode; + return c->back_labels[-label] - from; + } else { + /* Forward local label: prepend to (possibly-empty) linked list. */ + int *lptr = &c->fwd_labels[label]; + int32_t ret = (*lptr == EMPTYLABEL) ? 0 : *lptr - pcofs(c); + *lptr = pcofs(c); + return ret; + } +} + +static void put32(compiler *c, uint32_t v) { + mgroup *g = c->group; + if (c->pc == g->bytecode_end) { + int ofs = pcofs(c); + size_t oldsize = g->bytecode_end - g->bytecode; + size_t newsize = UPB_MAX(oldsize * 2, 64); + /* TODO(haberman): handle OOM. */ + g->bytecode = upb_grealloc(g->bytecode, oldsize * sizeof(uint32_t), + newsize * sizeof(uint32_t)); + g->bytecode_end = g->bytecode + newsize; + c->pc = g->bytecode + ofs; + } + *c->pc++ = v; +} + +static void putop(compiler *c, int op, ...) { + va_list ap; + va_start(ap, op); + + switch (op) { + case OP_SETDISPATCH: { + uintptr_t ptr = (uintptr_t)va_arg(ap, void*); + put32(c, OP_SETDISPATCH); + put32(c, ptr); + if (sizeof(uintptr_t) > sizeof(uint32_t)) + put32(c, (uint64_t)ptr >> 32); + break; + } + case OP_STARTMSG: + case OP_ENDMSG: + case OP_PUSHLENDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_HALT: + case OP_RET: + case OP_DISPATCH: + put32(c, op); + break; + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_STRING: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + put32(c, op | va_arg(ap, upb_selector_t) << 8); + break; + case OP_SETBIGGROUPNUM: + put32(c, op); + put32(c, va_arg(ap, int)); + break; + case OP_CALL: { + const upb_pbdecodermethod *method = va_arg(ap, upb_pbdecodermethod *); + put32(c, op | (method->code_base.ofs - (pcofs(c) + 1)) << 8); + break; + } + case OP_CHECKDELIM: + case OP_BRANCH: { + uint32_t instruction = op; + int label = va_arg(ap, int); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + break; + } + case OP_TAG1: + case OP_TAG2: { + int label = va_arg(ap, int); + uint64_t tag = va_arg(ap, uint64_t); + uint32_t instruction = op | (tag << 16); + UPB_ASSERT(tag <= 0xffff); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + break; + } + case OP_TAGN: { + int label = va_arg(ap, int); + uint64_t tag = va_arg(ap, uint64_t); + uint32_t instruction = op | (upb_value_size(tag) << 16); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + put32(c, tag); + put32(c, tag >> 32); + break; + } + } + + va_end(ap); +} + +#if defined(UPB_DUMP_BYTECODE) + +const char *upb_pbdecoder_getopname(unsigned int op) { +#define QUOTE(x) #x +#define EXPAND_AND_QUOTE(x) QUOTE(x) +#define OPNAME(x) OP_##x +#define OP(x) case OPNAME(x): return EXPAND_AND_QUOTE(OPNAME(x)); +#define T(x) OP(PARSE_##x) + /* Keep in sync with list in decoder.int.h. */ + switch ((opcode)op) { + T(DOUBLE) T(FLOAT) T(INT64) T(UINT64) T(INT32) T(FIXED64) T(FIXED32) + T(BOOL) T(UINT32) T(SFIXED32) T(SFIXED64) T(SINT32) T(SINT64) + OP(STARTMSG) OP(ENDMSG) OP(STARTSEQ) OP(ENDSEQ) OP(STARTSUBMSG) + OP(ENDSUBMSG) OP(STARTSTR) OP(STRING) OP(ENDSTR) OP(CALL) OP(RET) + OP(PUSHLENDELIM) OP(PUSHTAGDELIM) OP(SETDELIM) OP(CHECKDELIM) + OP(BRANCH) OP(TAG1) OP(TAG2) OP(TAGN) OP(SETDISPATCH) OP(POP) + OP(SETBIGGROUPNUM) OP(DISPATCH) OP(HALT) + } + return ""; +#undef OP +#undef T +} + +#endif + +#ifdef UPB_DUMP_BYTECODE + +static void dumpbc(uint32_t *p, uint32_t *end, FILE *f) { + + uint32_t *begin = p; + + while (p < end) { + fprintf(f, "%p %8tx", p, p - begin); + uint32_t instr = *p++; + uint8_t op = getop(instr); + fprintf(f, " %s", upb_pbdecoder_getopname(op)); + switch ((opcode)op) { + case OP_SETDISPATCH: { + const upb_inttable *dispatch; + memcpy(&dispatch, p, sizeof(void*)); + p += ptr_words; + const upb_pbdecodermethod *method = + (void *)((char *)dispatch - + offsetof(upb_pbdecodermethod, dispatch)); + fprintf(f, " %s", upb_msgdef_fullname( + upb_handlers_msgdef(method->dest_handlers_))); + break; + } + case OP_DISPATCH: + case OP_STARTMSG: + case OP_ENDMSG: + case OP_PUSHLENDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_HALT: + case OP_RET: + break; + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_STRING: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + fprintf(f, " %d", instr >> 8); + break; + case OP_SETBIGGROUPNUM: + fprintf(f, " %d", *p++); + break; + case OP_CHECKDELIM: + case OP_CALL: + case OP_BRANCH: + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + break; + case OP_TAG1: + case OP_TAG2: { + fprintf(f, " tag:0x%x", instr >> 16); + if (getofs(instr)) { + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + } + break; + } + case OP_TAGN: { + uint64_t tag = *p++; + tag |= (uint64_t)*p++ << 32; + fprintf(f, " tag:0x%llx", (long long)tag); + fprintf(f, " n:%d", instr >> 16); + if (getofs(instr)) { + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + } + break; + } + } + fputs("\n", f); + } +} + +#endif + +static uint64_t get_encoded_tag(const upb_fielddef *f, int wire_type) { + uint32_t tag = (upb_fielddef_number(f) << 3) | wire_type; + uint64_t encoded_tag = upb_vencode32(tag); + /* No tag should be greater than 5 bytes. */ + UPB_ASSERT(encoded_tag <= 0xffffffffff); + return encoded_tag; +} + +static void putchecktag(compiler *c, const upb_fielddef *f, + int wire_type, int dest) { + uint64_t tag = get_encoded_tag(f, wire_type); + switch (upb_value_size(tag)) { + case 1: + putop(c, OP_TAG1, dest, tag); + break; + case 2: + putop(c, OP_TAG2, dest, tag); + break; + default: + putop(c, OP_TAGN, dest, tag); + break; + } +} + +static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) { + upb_selector_t selector; + bool ok = upb_handlers_getselector(f, type, &selector); + UPB_ASSERT(ok); + return selector; +} + +/* Takes an existing, primary dispatch table entry and repacks it with a + * different alternate wire type. Called when we are inserting a secondary + * dispatch table entry for an alternate wire type. */ +static uint64_t repack(uint64_t dispatch, int new_wt2) { + uint64_t ofs; + uint8_t wt1; + uint8_t old_wt2; + upb_pbdecoder_unpackdispatch(dispatch, &ofs, &wt1, &old_wt2); + UPB_ASSERT(old_wt2 == NO_WIRE_TYPE); /* wt2 should not be set yet. */ + return upb_pbdecoder_packdispatch(ofs, wt1, new_wt2); +} + +/* Marks the current bytecode position as the dispatch target for this message, + * field, and wire type. */ +static void dispatchtarget(compiler *c, upb_pbdecodermethod *method, + const upb_fielddef *f, int wire_type) { + /* Offset is relative to msg base. */ + uint64_t ofs = pcofs(c) - method->code_base.ofs; + uint32_t fn = upb_fielddef_number(f); + upb_inttable *d = &method->dispatch; + upb_value v; + if (upb_inttable_remove(d, fn, &v)) { + /* TODO: prioritize based on packed setting in .proto file. */ + uint64_t repacked = repack(upb_value_getuint64(v), wire_type); + upb_inttable_insert(d, fn, upb_value_uint64(repacked)); + upb_inttable_insert(d, fn + UPB_MAX_FIELDNUMBER, upb_value_uint64(ofs)); + } else { + uint64_t val = upb_pbdecoder_packdispatch(ofs, wire_type, NO_WIRE_TYPE); + upb_inttable_insert(d, fn, upb_value_uint64(val)); + } +} + +static void putpush(compiler *c, const upb_fielddef *f) { + if (upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE) { + putop(c, OP_PUSHLENDELIM); + } else { + uint32_t fn = upb_fielddef_number(f); + if (fn >= 1 << 24) { + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_SETBIGGROUPNUM, fn); + } else { + putop(c, OP_PUSHTAGDELIM, fn); + } + } +} + +static upb_pbdecodermethod *find_submethod(const compiler *c, + const upb_pbdecodermethod *method, + const upb_fielddef *f) { + const upb_handlers *sub = + upb_handlers_getsubhandlers(method->dest_handlers_, f); + upb_value v; + return upb_inttable_lookupptr(&c->group->methods, sub, &v) + ? upb_value_getptr(v) + : NULL; +} + +static void putsel(compiler *c, opcode op, upb_selector_t sel, + const upb_handlers *h) { + if (upb_handlers_gethandler(h, sel, NULL)) { + putop(c, op, sel); + } +} + +/* Puts an opcode to call a callback, but only if a callback actually exists for + * this field and handler type. */ +static void maybeput(compiler *c, opcode op, const upb_handlers *h, + const upb_fielddef *f, upb_handlertype_t type) { + putsel(c, op, getsel(f, type), h); +} + +static bool haslazyhandlers(const upb_handlers *h, const upb_fielddef *f) { + if (!upb_fielddef_lazy(f)) + return false; + + return upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STARTSTR), NULL) || + upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STRING), NULL) || + upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_ENDSTR), NULL); +} + + +/* bytecode compiler code generation ******************************************/ + +/* Symbolic names for our local labels. */ +#define LABEL_LOOPSTART 1 /* Top of a repeated field loop. */ +#define LABEL_LOOPBREAK 2 /* To jump out of a repeated loop */ +#define LABEL_FIELD 3 /* Jump backward to find the most recent field. */ +#define LABEL_ENDMSG 4 /* To reach the OP_ENDMSG instr for this msg. */ + +/* Generates bytecode to parse a single non-lazy message field. */ +static void generate_msgfield(compiler *c, const upb_fielddef *f, + upb_pbdecodermethod *method) { + const upb_handlers *h = upb_pbdecodermethod_desthandlers(method); + const upb_pbdecodermethod *sub_m = find_submethod(c, method, f); + int wire_type; + + if (!sub_m) { + /* Don't emit any code for this field at all; it will be parsed as an + * unknown field. + * + * TODO(haberman): we should change this to parse it as a string field + * instead. It will probably be faster, but more importantly, once we + * start vending unknown fields, a field shouldn't be treated as unknown + * just because it doesn't have subhandlers registered. */ + return; + } + + label(c, LABEL_FIELD); + + wire_type = + (upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE) + ? UPB_WIRE_TYPE_DELIMITED + : UPB_WIRE_TYPE_START_GROUP; + + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); + label(c, LABEL_LOOPSTART); + putpush(c, f); + putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG)); + putop(c, OP_CALL, sub_m); + putop(c, OP_POP); + maybeput(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG); + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + putop(c, OP_SETDELIM); + } + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, wire_type, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); + maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ); + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putpush(c, f); + putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG)); + putop(c, OP_CALL, sub_m); + putop(c, OP_POP); + maybeput(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG); + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + putop(c, OP_SETDELIM); + } + } +} + +/* Generates bytecode to parse a single string or lazy submessage field. */ +static void generate_delimfield(compiler *c, const upb_fielddef *f, + upb_pbdecodermethod *method) { + const upb_handlers *h = upb_pbdecodermethod_desthandlers(method); + + label(c, LABEL_FIELD); + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); + label(c, LABEL_LOOPSTART); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR)); + /* Need to emit even if no handler to skip past the string. */ + putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING)); + maybeput(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR); + putop(c, OP_POP); + putop(c, OP_SETDELIM); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); + maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ); + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR)); + putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING)); + maybeput(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR); + putop(c, OP_POP); + putop(c, OP_SETDELIM); + } +} + +/* Generates bytecode to parse a single primitive field. */ +static void generate_primitivefield(compiler *c, const upb_fielddef *f, + upb_pbdecodermethod *method) { + const upb_handlers *h = upb_pbdecodermethod_desthandlers(method); + upb_descriptortype_t descriptor_type = upb_fielddef_descriptortype(f); + opcode parse_type; + upb_selector_t sel; + int wire_type; + + label(c, LABEL_FIELD); + + /* From a decoding perspective, ENUM is the same as INT32. */ + if (descriptor_type == UPB_DESCRIPTOR_TYPE_ENUM) + descriptor_type = UPB_DESCRIPTOR_TYPE_INT32; + + parse_type = (opcode)descriptor_type; + + /* TODO(haberman): generate packed or non-packed first depending on "packed" + * setting in the fielddef. This will favor (in speed) whichever was + * specified. */ + + UPB_ASSERT((int)parse_type >= 0 && parse_type <= OP_MAX); + sel = getsel(f, upb_handlers_getprimitivehandlertype(f)); + wire_type = upb_pb_native_wire_types[upb_fielddef_descriptortype(f)]; + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); /* Packed */ + label(c, LABEL_LOOPSTART); + putop(c, parse_type, sel); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + dispatchtarget(c, method, f, wire_type); + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); /* Non-packed */ + label(c, LABEL_LOOPSTART); + putop(c, parse_type, sel); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, wire_type, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); /* Packed and non-packed join. */ + maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ); + putop(c, OP_SETDELIM); /* Could remove for non-packed by dup ENDSEQ. */ + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putop(c, parse_type, sel); + } +} + +/* Adds bytecode for parsing the given message to the given decoderplan, + * while adding all dispatch targets to this message's dispatch table. */ +static void compile_method(compiler *c, upb_pbdecodermethod *method) { + const upb_handlers *h; + const upb_msgdef *md; + uint32_t* start_pc; + upb_msg_field_iter i; + upb_value val; + + UPB_ASSERT(method); + + /* Clear all entries in the dispatch table. */ + upb_inttable_uninit(&method->dispatch); + upb_inttable_init(&method->dispatch, UPB_CTYPE_UINT64); + + h = upb_pbdecodermethod_desthandlers(method); + md = upb_handlers_msgdef(h); + + method->code_base.ofs = pcofs(c); + putop(c, OP_SETDISPATCH, &method->dispatch); + putsel(c, OP_STARTMSG, UPB_STARTMSG_SELECTOR, h); + label(c, LABEL_FIELD); + start_pc = c->pc; + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_fieldtype_t type = upb_fielddef_type(f); + + if (type == UPB_TYPE_MESSAGE && !(haslazyhandlers(h, f) && c->lazy)) { + generate_msgfield(c, f, method); + } else if (type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES || + type == UPB_TYPE_MESSAGE) { + generate_delimfield(c, f, method); + } else { + generate_primitivefield(c, f, method); + } + } + + /* If there were no fields, or if no handlers were defined, we need to + * generate a non-empty loop body so that we can at least dispatch for unknown + * fields and check for the end of the message. */ + if (c->pc == start_pc) { + /* Check for end-of-message. */ + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + /* Unconditionally dispatch. */ + putop(c, OP_DISPATCH, 0); + } + + /* For now we just loop back to the last field of the message (or if none, + * the DISPATCH opcode for the message). */ + putop(c, OP_BRANCH, -LABEL_FIELD); + + /* Insert both a label and a dispatch table entry for this end-of-msg. */ + label(c, LABEL_ENDMSG); + val = upb_value_uint64(pcofs(c) - method->code_base.ofs); + upb_inttable_insert(&method->dispatch, DISPATCH_ENDMSG, val); + + putsel(c, OP_ENDMSG, UPB_ENDMSG_SELECTOR, h); + putop(c, OP_RET); + + upb_inttable_compact(&method->dispatch); +} + +/* Populate "methods" with new upb_pbdecodermethod objects reachable from "h". + * Returns the method for these handlers. + * + * Generates a new method for every destination handlers reachable from "h". */ +static void find_methods(compiler *c, const upb_handlers *h) { + upb_value v; + upb_msg_field_iter i; + const upb_msgdef *md; + upb_pbdecodermethod *method; + + if (upb_inttable_lookupptr(&c->group->methods, h, &v)) + return; + + method = newmethod(h, c->group); + upb_inttable_insertptr(&c->group->methods, h, upb_value_ptr(method)); + + /* Find submethods. */ + md = upb_handlers_msgdef(h); + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + const upb_handlers *sub_h; + if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE && + (sub_h = upb_handlers_getsubhandlers(h, f)) != NULL) { + /* We only generate a decoder method for submessages with handlers. + * Others will be parsed as unknown fields. */ + find_methods(c, sub_h); + } + } +} + +/* (Re-)compile bytecode for all messages in "msgs." + * Overwrites any existing bytecode in "c". */ +static void compile_methods(compiler *c) { + upb_inttable_iter i; + + /* Start over at the beginning of the bytecode. */ + c->pc = c->group->bytecode; + + upb_inttable_begin(&i, &c->group->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); + compile_method(c, method); + } +} + +static void set_bytecode_handlers(mgroup *g) { + upb_inttable_iter i; + upb_inttable_begin(&i, &g->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *m = upb_value_getptr(upb_inttable_iter_value(&i)); + upb_byteshandler *h = &m->input_handler_; + + m->code_base.ptr = g->bytecode + m->code_base.ofs; + + upb_byteshandler_setstartstr(h, upb_pbdecoder_startbc, m->code_base.ptr); + upb_byteshandler_setstring(h, upb_pbdecoder_decode, g); + upb_byteshandler_setendstr(h, upb_pbdecoder_end, m); + } +} + + +/* TODO(haberman): allow this to be constructed for an arbitrary set of dest + * handlers and other mgroups (but verify we have a transitive closure). */ +const mgroup *mgroup_new(const upb_handlers *dest, bool lazy) { + mgroup *g; + compiler *c; + + g = newgroup(); + c = newcompiler(g, lazy); + find_methods(c, dest); + + /* We compile in two passes: + * 1. all messages are assigned relative offsets from the beginning of the + * bytecode (saved in method->code_base). + * 2. forwards OP_CALL instructions can be correctly linked since message + * offsets have been previously assigned. + * + * Could avoid the second pass by linking OP_CALL instructions somehow. */ + compile_methods(c); + compile_methods(c); + g->bytecode_end = c->pc; + freecompiler(c); + +#ifdef UPB_DUMP_BYTECODE + { + FILE *f = fopen("/tmp/upb-bytecode", "w"); + UPB_ASSERT(f); + dumpbc(g->bytecode, g->bytecode_end, stderr); + dumpbc(g->bytecode, g->bytecode_end, f); + fclose(f); + + f = fopen("/tmp/upb-bytecode.bin", "wb"); + UPB_ASSERT(f); + fwrite(g->bytecode, 1, g->bytecode_end - g->bytecode, f); + fclose(f); + } +#endif + + set_bytecode_handlers(g); + return g; +} + + +/* upb_pbcodecache ************************************************************/ + +upb_pbcodecache *upb_pbcodecache_new(upb_handlercache *dest) { + upb_pbcodecache *c = upb_gmalloc(sizeof(*c)); + + if (!c) return NULL; + + c->dest = dest; + c->lazy = false; + + c->arena = upb_arena_new(); + if (!upb_inttable_init(&c->groups, UPB_CTYPE_CONSTPTR)) return NULL; + + return c; +} + +void upb_pbcodecache_free(upb_pbcodecache *c) { + upb_inttable_iter i; + + upb_inttable_begin(&i, &c->groups); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_value val = upb_inttable_iter_value(&i); + freegroup((void*)upb_value_getconstptr(val)); + } + + upb_inttable_uninit(&c->groups); + upb_arena_free(c->arena); + upb_gfree(c); +} + +void upb_pbdecodermethodopts_setlazy(upb_pbcodecache *c, bool lazy) { + UPB_ASSERT(upb_inttable_count(&c->groups) == 0); + c->lazy = lazy; +} + +const upb_pbdecodermethod *upb_pbcodecache_get(upb_pbcodecache *c, + const upb_msgdef *md) { + upb_value v; + bool ok; + const upb_handlers *h; + const mgroup *g; + + h = upb_handlercache_get(c->dest, md); + if (upb_inttable_lookupptr(&c->groups, md, &v)) { + g = upb_value_getconstptr(v); + } else { + g = mgroup_new(h, c->lazy); + ok = upb_inttable_insertptr(&c->groups, md, upb_value_constptr(g)); + UPB_ASSERT(ok); + } + + ok = upb_inttable_lookupptr(&g->methods, h, &v); + UPB_ASSERT(ok); + return upb_value_getptr(v); +} diff --git a/third_party/upb/upb/pb/decoder.c b/third_party/upb/upb/pb/decoder.c new file mode 100644 index 00000000000..735bef18f8e --- /dev/null +++ b/third_party/upb/upb/pb/decoder.c @@ -0,0 +1,1050 @@ +/* +** upb::Decoder (Bytecode Decoder VM) +** +** Bytecode must previously have been generated using the bytecode compiler in +** compile_decoder.c. This decoder then walks through the bytecode op-by-op to +** parse the input. +** +** Decoding is fully resumable; we just keep a pointer to the current bytecode +** instruction and resume from there. A fair amount of the logic here is to +** handle the fact that values can span buffer seams and we have to be able to +** be capable of suspending/resuming from any byte in the stream. This +** sometimes requires keeping a few trailing bytes from the last buffer around +** in the "residual" buffer. +*/ + +#include +#include +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" + +#ifdef UPB_DUMP_BYTECODE +#include +#endif + +#include "upb/port_def.inc" + +#define CHECK_SUSPEND(x) if (!(x)) return upb_pbdecoder_suspend(d); + +/* Error messages that are shared between the bytecode and JIT decoders. */ +const char *kPbDecoderStackOverflow = "Nesting too deep."; +const char *kPbDecoderSubmessageTooLong = + "Submessage end extends past enclosing submessage."; + +/* Error messages shared within this file. */ +static const char *kUnterminatedVarint = "Unterminated varint."; + +/* upb_pbdecoder **************************************************************/ + +static opcode halt = OP_HALT; + +/* A dummy character we can point to when the user passes us a NULL buffer. + * We need this because in C (NULL + 0) and (NULL - NULL) are undefined + * behavior, which would invalidate functions like curbufleft(). */ +static const char dummy_char; + +/* Whether an op consumes any of the input buffer. */ +static bool consumes_input(opcode op) { + switch (op) { + case OP_SETDISPATCH: + case OP_STARTMSG: + case OP_ENDMSG: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_SETBIGGROUPNUM: + case OP_CHECKDELIM: + case OP_CALL: + case OP_RET: + case OP_BRANCH: + return false; + default: + return true; + } +} + +static size_t stacksize(upb_pbdecoder *d, size_t entries) { + UPB_UNUSED(d); + return entries * sizeof(upb_pbdecoder_frame); +} + +static size_t callstacksize(upb_pbdecoder *d, size_t entries) { + UPB_UNUSED(d); + + return entries * sizeof(uint32_t*); +} + + +static bool in_residual_buf(const upb_pbdecoder *d, const char *p); + +/* It's unfortunate that we have to micro-manage the compiler with + * UPB_FORCEINLINE and UPB_NOINLINE, especially since this tuning is necessarily + * specific to one hardware configuration. But empirically on a Core i7, + * performance increases 30-50% with these annotations. Every instance where + * these appear, gcc 4.2.1 made the wrong decision and degraded performance in + * benchmarks. */ + +static void seterr(upb_pbdecoder *d, const char *msg) { + upb_status_seterrmsg(d->status, msg); +} + +void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg) { + seterr(d, msg); +} + + +/* Buffering ******************************************************************/ + +/* We operate on one buffer at a time, which is either the user's buffer passed + * to our "decode" callback or some residual bytes from the previous buffer. */ + +/* How many bytes can be safely read from d->ptr without reading past end-of-buf + * or past the current delimited end. */ +static size_t curbufleft(const upb_pbdecoder *d) { + UPB_ASSERT(d->data_end >= d->ptr); + return d->data_end - d->ptr; +} + +/* How many bytes are available before end-of-buffer. */ +static size_t bufleft(const upb_pbdecoder *d) { + return d->end - d->ptr; +} + +/* Overall stream offset of d->ptr. */ +uint64_t offset(const upb_pbdecoder *d) { + return d->bufstart_ofs + (d->ptr - d->buf); +} + +/* How many bytes are available before the end of this delimited region. */ +size_t delim_remaining(const upb_pbdecoder *d) { + return d->top->end_ofs - offset(d); +} + +/* Advances d->ptr. */ +static void advance(upb_pbdecoder *d, size_t len) { + UPB_ASSERT(curbufleft(d) >= len); + d->ptr += len; +} + +static bool in_buf(const char *p, const char *buf, const char *end) { + return p >= buf && p <= end; +} + +static bool in_residual_buf(const upb_pbdecoder *d, const char *p) { + return in_buf(p, d->residual, d->residual_end); +} + +/* Calculates the delim_end value, which is affected by both the current buffer + * and the parsing stack, so must be called whenever either is updated. */ +static void set_delim_end(upb_pbdecoder *d) { + size_t delim_ofs = d->top->end_ofs - d->bufstart_ofs; + if (delim_ofs <= (size_t)(d->end - d->buf)) { + d->delim_end = d->buf + delim_ofs; + d->data_end = d->delim_end; + } else { + d->data_end = d->end; + d->delim_end = NULL; + } +} + +static void switchtobuf(upb_pbdecoder *d, const char *buf, const char *end) { + d->ptr = buf; + d->buf = buf; + d->end = end; + set_delim_end(d); +} + +static void advancetobuf(upb_pbdecoder *d, const char *buf, size_t len) { + UPB_ASSERT(curbufleft(d) == 0); + d->bufstart_ofs += (d->end - d->buf); + switchtobuf(d, buf, buf + len); +} + +static void checkpoint(upb_pbdecoder *d) { + /* The assertion here is in the interests of efficiency, not correctness. + * We are trying to ensure that we don't checkpoint() more often than + * necessary. */ + UPB_ASSERT(d->checkpoint != d->ptr); + d->checkpoint = d->ptr; +} + +/* Skips "bytes" bytes in the stream, which may be more than available. If we + * skip more bytes than are available, we return a long read count to the caller + * indicating how many bytes can be skipped over before passing actual data + * again. Skipped bytes can pass a NULL buffer and the decoder guarantees they + * won't actually be read. + */ +static int32_t skip(upb_pbdecoder *d, size_t bytes) { + UPB_ASSERT(!in_residual_buf(d, d->ptr) || d->size_param == 0); + UPB_ASSERT(d->skip == 0); + if (bytes > delim_remaining(d)) { + seterr(d, "Skipped value extended beyond enclosing submessage."); + return upb_pbdecoder_suspend(d); + } else if (bufleft(d) >= bytes) { + /* Skipped data is all in current buffer, and more is still available. */ + advance(d, bytes); + d->skip = 0; + return DECODE_OK; + } else { + /* Skipped data extends beyond currently available buffers. */ + d->pc = d->last; + d->skip = bytes - curbufleft(d); + d->bufstart_ofs += (d->end - d->buf); + d->residual_end = d->residual; + switchtobuf(d, d->residual, d->residual_end); + return d->size_param + d->skip; + } +} + + +/* Resumes the decoder from an initial state or from a previous suspend. */ +int32_t upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf, + size_t size, const upb_bufhandle *handle) { + UPB_UNUSED(p); /* Useless; just for the benefit of the JIT. */ + + /* d->skip and d->residual_end could probably elegantly be represented + * as a single variable, to more easily represent this invariant. */ + UPB_ASSERT(!(d->skip && d->residual_end > d->residual)); + + /* We need to remember the original size_param, so that the value we return + * is relative to it, even if we do some skipping first. */ + d->size_param = size; + d->handle = handle; + + /* Have to handle this case specially (ie. not with skip()) because the user + * is allowed to pass a NULL buffer here, which won't allow us to safely + * calculate a d->end or use our normal functions like curbufleft(). */ + if (d->skip && d->skip >= size) { + d->skip -= size; + d->bufstart_ofs += size; + buf = &dummy_char; + size = 0; + + /* We can't just return now, because we might need to execute some ops + * like CHECKDELIM, which could call some callbacks and pop the stack. */ + } + + /* We need to pretend that this was the actual buffer param, since some of the + * calculations assume that d->ptr/d->buf is relative to this. */ + d->buf_param = buf; + + if (!buf) { + /* NULL buf is ok if its entire span is covered by the "skip" above, but + * by this point we know that "skip" doesn't cover the buffer. */ + seterr(d, "Passed NULL buffer over non-skippable region."); + return upb_pbdecoder_suspend(d); + } + + if (d->residual_end > d->residual) { + /* We have residual bytes from the last buffer. */ + UPB_ASSERT(d->ptr == d->residual); + } else { + switchtobuf(d, buf, buf + size); + } + + d->checkpoint = d->ptr; + + /* Handle skips that don't cover the whole buffer (as above). */ + if (d->skip) { + size_t skip_bytes = d->skip; + d->skip = 0; + CHECK_RETURN(skip(d, skip_bytes)); + checkpoint(d); + } + + /* If we're inside an unknown group, continue to parse unknown values. */ + if (d->top->groupnum < 0) { + CHECK_RETURN(upb_pbdecoder_skipunknown(d, -1, 0)); + checkpoint(d); + } + + return DECODE_OK; +} + +/* Suspends the decoder at the last checkpoint, without saving any residual + * bytes. If there are any unconsumed bytes, returns a short byte count. */ +size_t upb_pbdecoder_suspend(upb_pbdecoder *d) { + d->pc = d->last; + if (d->checkpoint == d->residual) { + /* Checkpoint was in residual buf; no user bytes were consumed. */ + d->ptr = d->residual; + return 0; + } else { + size_t ret = d->size_param - (d->end - d->checkpoint); + UPB_ASSERT(!in_residual_buf(d, d->checkpoint)); + UPB_ASSERT(d->buf == d->buf_param || d->buf == &dummy_char); + + d->bufstart_ofs += (d->checkpoint - d->buf); + d->residual_end = d->residual; + switchtobuf(d, d->residual, d->residual_end); + return ret; + } +} + +/* Suspends the decoder at the last checkpoint, and saves any unconsumed + * bytes in our residual buffer. This is necessary if we need more user + * bytes to form a complete value, which might not be contiguous in the + * user's buffers. Always consumes all user bytes. */ +static size_t suspend_save(upb_pbdecoder *d) { + /* We hit end-of-buffer before we could parse a full value. + * Save any unconsumed bytes (if any) to the residual buffer. */ + d->pc = d->last; + + if (d->checkpoint == d->residual) { + /* Checkpoint was in residual buf; append user byte(s) to residual buf. */ + UPB_ASSERT((d->residual_end - d->residual) + d->size_param <= + sizeof(d->residual)); + if (!in_residual_buf(d, d->ptr)) { + d->bufstart_ofs -= (d->residual_end - d->residual); + } + memcpy(d->residual_end, d->buf_param, d->size_param); + d->residual_end += d->size_param; + } else { + /* Checkpoint was in user buf; old residual bytes not needed. */ + size_t save; + UPB_ASSERT(!in_residual_buf(d, d->checkpoint)); + + d->ptr = d->checkpoint; + save = curbufleft(d); + UPB_ASSERT(save <= sizeof(d->residual)); + memcpy(d->residual, d->ptr, save); + d->residual_end = d->residual + save; + d->bufstart_ofs = offset(d); + } + + switchtobuf(d, d->residual, d->residual_end); + return d->size_param; +} + +/* Copies the next "bytes" bytes into "buf" and advances the stream. + * Requires that this many bytes are available in the current buffer. */ +UPB_FORCEINLINE static void consumebytes(upb_pbdecoder *d, void *buf, + size_t bytes) { + UPB_ASSERT(bytes <= curbufleft(d)); + memcpy(buf, d->ptr, bytes); + advance(d, bytes); +} + +/* Slow path for getting the next "bytes" bytes, regardless of whether they are + * available in the current buffer or not. Returns a status code as described + * in decoder.int.h. */ +UPB_NOINLINE static int32_t getbytes_slow(upb_pbdecoder *d, void *buf, + size_t bytes) { + const size_t avail = curbufleft(d); + consumebytes(d, buf, avail); + bytes -= avail; + UPB_ASSERT(bytes > 0); + if (in_residual_buf(d, d->ptr)) { + advancetobuf(d, d->buf_param, d->size_param); + } + if (curbufleft(d) >= bytes) { + consumebytes(d, (char *)buf + avail, bytes); + return DECODE_OK; + } else if (d->data_end == d->delim_end) { + seterr(d, "Submessage ended in the middle of a value or group"); + return upb_pbdecoder_suspend(d); + } else { + return suspend_save(d); + } +} + +/* Gets the next "bytes" bytes, regardless of whether they are available in the + * current buffer or not. Returns a status code as described in decoder.int.h. + */ +UPB_FORCEINLINE static int32_t getbytes(upb_pbdecoder *d, void *buf, + size_t bytes) { + if (curbufleft(d) >= bytes) { + /* Buffer has enough data to satisfy. */ + consumebytes(d, buf, bytes); + return DECODE_OK; + } else { + return getbytes_slow(d, buf, bytes); + } +} + +UPB_NOINLINE static size_t peekbytes_slow(upb_pbdecoder *d, void *buf, + size_t bytes) { + size_t ret = curbufleft(d); + memcpy(buf, d->ptr, ret); + if (in_residual_buf(d, d->ptr)) { + size_t copy = UPB_MIN(bytes - ret, d->size_param); + memcpy((char *)buf + ret, d->buf_param, copy); + ret += copy; + } + return ret; +} + +UPB_FORCEINLINE static size_t peekbytes(upb_pbdecoder *d, void *buf, + size_t bytes) { + if (curbufleft(d) >= bytes) { + memcpy(buf, d->ptr, bytes); + return bytes; + } else { + return peekbytes_slow(d, buf, bytes); + } +} + + +/* Decoding of wire types *****************************************************/ + +/* Slow path for decoding a varint from the current buffer position. + * Returns a status code as described in decoder.int.h. */ +UPB_NOINLINE int32_t upb_pbdecoder_decode_varint_slow(upb_pbdecoder *d, + uint64_t *u64) { + uint8_t byte = 0x80; + int bitpos; + *u64 = 0; + for(bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { + CHECK_RETURN(getbytes(d, &byte, 1)); + *u64 |= (uint64_t)(byte & 0x7F) << bitpos; + } + if(bitpos == 70 && (byte & 0x80)) { + seterr(d, kUnterminatedVarint); + return upb_pbdecoder_suspend(d); + } + return DECODE_OK; +} + +/* Decodes a varint from the current buffer position. + * Returns a status code as described in decoder.int.h. */ +UPB_FORCEINLINE static int32_t decode_varint(upb_pbdecoder *d, uint64_t *u64) { + if (curbufleft(d) > 0 && !(*d->ptr & 0x80)) { + *u64 = *d->ptr; + advance(d, 1); + return DECODE_OK; + } else if (curbufleft(d) >= 10) { + /* Fast case. */ + upb_decoderet r = upb_vdecode_fast(d->ptr); + if (r.p == NULL) { + seterr(d, kUnterminatedVarint); + return upb_pbdecoder_suspend(d); + } + advance(d, r.p - d->ptr); + *u64 = r.val; + return DECODE_OK; + } else { + /* Slow case -- varint spans buffer seam. */ + return upb_pbdecoder_decode_varint_slow(d, u64); + } +} + +/* Decodes a 32-bit varint from the current buffer position. + * Returns a status code as described in decoder.int.h. */ +UPB_FORCEINLINE static int32_t decode_v32(upb_pbdecoder *d, uint32_t *u32) { + uint64_t u64; + int32_t ret = decode_varint(d, &u64); + if (ret >= 0) return ret; + if (u64 > UINT32_MAX) { + seterr(d, "Unterminated 32-bit varint"); + /* TODO(haberman) guarantee that this function return is >= 0 somehow, + * so we know this path will always be treated as error by our caller. + * Right now the size_t -> int32_t can overflow and produce negative values. + */ + *u32 = 0; + return upb_pbdecoder_suspend(d); + } + *u32 = u64; + return DECODE_OK; +} + +/* Decodes a fixed32 from the current buffer position. + * Returns a status code as described in decoder.int.h. + * TODO: proper byte swapping for big-endian machines. */ +UPB_FORCEINLINE static int32_t decode_fixed32(upb_pbdecoder *d, uint32_t *u32) { + return getbytes(d, u32, 4); +} + +/* Decodes a fixed64 from the current buffer position. + * Returns a status code as described in decoder.int.h. + * TODO: proper byte swapping for big-endian machines. */ +UPB_FORCEINLINE static int32_t decode_fixed64(upb_pbdecoder *d, uint64_t *u64) { + return getbytes(d, u64, 8); +} + +/* Non-static versions of the above functions. + * These are called by the JIT for fallback paths. */ +int32_t upb_pbdecoder_decode_f32(upb_pbdecoder *d, uint32_t *u32) { + return decode_fixed32(d, u32); +} + +int32_t upb_pbdecoder_decode_f64(upb_pbdecoder *d, uint64_t *u64) { + return decode_fixed64(d, u64); +} + +static double as_double(uint64_t n) { double d; memcpy(&d, &n, 8); return d; } +static float as_float(uint32_t n) { float f; memcpy(&f, &n, 4); return f; } + +/* Pushes a frame onto the decoder stack. */ +static bool decoder_push(upb_pbdecoder *d, uint64_t end) { + upb_pbdecoder_frame *fr = d->top; + + if (end > fr->end_ofs) { + seterr(d, kPbDecoderSubmessageTooLong); + return false; + } else if (fr == d->limit) { + seterr(d, kPbDecoderStackOverflow); + return false; + } + + fr++; + fr->end_ofs = end; + fr->dispatch = NULL; + fr->groupnum = 0; + d->top = fr; + return true; +} + +static bool pushtagdelim(upb_pbdecoder *d, uint32_t arg) { + /* While we expect to see an "end" tag (either ENDGROUP or a non-sequence + * field number) prior to hitting any enclosing submessage end, pushing our + * existing delim end prevents us from continuing to parse values from a + * corrupt proto that doesn't give us an END tag in time. */ + if (!decoder_push(d, d->top->end_ofs)) + return false; + d->top->groupnum = arg; + return true; +} + +/* Pops a frame from the decoder stack. */ +static void decoder_pop(upb_pbdecoder *d) { d->top--; } + +UPB_NOINLINE int32_t upb_pbdecoder_checktag_slow(upb_pbdecoder *d, + uint64_t expected) { + uint64_t data = 0; + size_t bytes = upb_value_size(expected); + size_t read = peekbytes(d, &data, bytes); + if (read == bytes && data == expected) { + /* Advance past matched bytes. */ + int32_t ok = getbytes(d, &data, read); + UPB_ASSERT(ok < 0); + return DECODE_OK; + } else if (read < bytes && memcmp(&data, &expected, read) == 0) { + return suspend_save(d); + } else { + return DECODE_MISMATCH; + } +} + +int32_t upb_pbdecoder_skipunknown(upb_pbdecoder *d, int32_t fieldnum, + uint8_t wire_type) { + if (fieldnum >= 0) + goto have_tag; + + while (true) { + uint32_t tag; + CHECK_RETURN(decode_v32(d, &tag)); + wire_type = tag & 0x7; + fieldnum = tag >> 3; + +have_tag: + if (fieldnum == 0) { + seterr(d, "Saw invalid field number (0)"); + return upb_pbdecoder_suspend(d); + } + + switch (wire_type) { + case UPB_WIRE_TYPE_32BIT: + CHECK_RETURN(skip(d, 4)); + break; + case UPB_WIRE_TYPE_64BIT: + CHECK_RETURN(skip(d, 8)); + break; + case UPB_WIRE_TYPE_VARINT: { + uint64_t u64; + CHECK_RETURN(decode_varint(d, &u64)); + break; + } + case UPB_WIRE_TYPE_DELIMITED: { + uint32_t len; + CHECK_RETURN(decode_v32(d, &len)); + CHECK_RETURN(skip(d, len)); + break; + } + case UPB_WIRE_TYPE_START_GROUP: + CHECK_SUSPEND(pushtagdelim(d, -fieldnum)); + break; + case UPB_WIRE_TYPE_END_GROUP: + if (fieldnum == -d->top->groupnum) { + decoder_pop(d); + } else if (fieldnum == d->top->groupnum) { + return DECODE_ENDGROUP; + } else { + seterr(d, "Unmatched ENDGROUP tag."); + return upb_pbdecoder_suspend(d); + } + break; + default: + seterr(d, "Invalid wire type"); + return upb_pbdecoder_suspend(d); + } + + if (d->top->groupnum >= 0) { + /* TODO: More code needed for handling unknown groups. */ + upb_sink_putunknown(d->top->sink, d->checkpoint, d->ptr - d->checkpoint); + return DECODE_OK; + } + + /* Unknown group -- continue looping over unknown fields. */ + checkpoint(d); + } +} + +static void goto_endmsg(upb_pbdecoder *d) { + upb_value v; + bool found = upb_inttable_lookup32(d->top->dispatch, DISPATCH_ENDMSG, &v); + UPB_ASSERT(found); + d->pc = d->top->base + upb_value_getuint64(v); +} + +/* Parses a tag and jumps to the corresponding bytecode instruction for this + * field. + * + * If the tag is unknown (or the wire type doesn't match), parses the field as + * unknown. If the tag is a valid ENDGROUP tag, jumps to the bytecode + * instruction for the end of message. */ +static int32_t dispatch(upb_pbdecoder *d) { + upb_inttable *dispatch = d->top->dispatch; + uint32_t tag; + uint8_t wire_type; + uint32_t fieldnum; + upb_value val; + int32_t retval; + + /* Decode tag. */ + CHECK_RETURN(decode_v32(d, &tag)); + wire_type = tag & 0x7; + fieldnum = tag >> 3; + + /* Lookup tag. Because of packed/non-packed compatibility, we have to + * check the wire type against two possibilities. */ + if (fieldnum != DISPATCH_ENDMSG && + upb_inttable_lookup32(dispatch, fieldnum, &val)) { + uint64_t v = upb_value_getuint64(val); + if (wire_type == (v & 0xff)) { + d->pc = d->top->base + (v >> 16); + return DECODE_OK; + } else if (wire_type == ((v >> 8) & 0xff)) { + bool found = + upb_inttable_lookup(dispatch, fieldnum + UPB_MAX_FIELDNUMBER, &val); + UPB_ASSERT(found); + d->pc = d->top->base + upb_value_getuint64(val); + return DECODE_OK; + } + } + + /* We have some unknown fields (or ENDGROUP) to parse. The DISPATCH or TAG + * bytecode that triggered this is preceded by a CHECKDELIM bytecode which + * we need to back up to, so that when we're done skipping unknown data we + * can re-check the delimited end. */ + d->last--; /* Necessary if we get suspended */ + d->pc = d->last; + UPB_ASSERT(getop(*d->last) == OP_CHECKDELIM); + + /* Unknown field or ENDGROUP. */ + retval = upb_pbdecoder_skipunknown(d, fieldnum, wire_type); + + CHECK_RETURN(retval); + + if (retval == DECODE_ENDGROUP) { + goto_endmsg(d); + return DECODE_OK; + } + + return DECODE_OK; +} + +/* Callers know that the stack is more than one deep because the opcodes that + * call this only occur after PUSH operations. */ +upb_pbdecoder_frame *outer_frame(upb_pbdecoder *d) { + UPB_ASSERT(d->top != d->stack); + return d->top - 1; +} + + +/* The main decoding loop *****************************************************/ + +/* The main decoder VM function. Uses traditional bytecode dispatch loop with a + * switch() statement. */ +size_t run_decoder_vm(upb_pbdecoder *d, const mgroup *group, + const upb_bufhandle* handle) { + +#define VMCASE(op, code) \ + case op: { code; if (consumes_input(op)) checkpoint(d); break; } +#define PRIMITIVE_OP(type, wt, name, convfunc, ctype) \ + VMCASE(OP_PARSE_ ## type, { \ + ctype val; \ + CHECK_RETURN(decode_ ## wt(d, &val)); \ + upb_sink_put ## name(d->top->sink, arg, (convfunc)(val)); \ + }) + + while(1) { + int32_t instruction; + opcode op; + uint32_t arg; + int32_t longofs; + + d->last = d->pc; + instruction = *d->pc++; + op = getop(instruction); + arg = instruction >> 8; + longofs = arg; + UPB_ASSERT(d->ptr != d->residual_end); + UPB_UNUSED(group); +#ifdef UPB_DUMP_BYTECODE + fprintf(stderr, "s_ofs=%d buf_ofs=%d data_rem=%d buf_rem=%d delim_rem=%d " + "%x %s (%d)\n", + (int)offset(d), + (int)(d->ptr - d->buf), + (int)(d->data_end - d->ptr), + (int)(d->end - d->ptr), + (int)((d->top->end_ofs - d->bufstart_ofs) - (d->ptr - d->buf)), + (int)(d->pc - 1 - group->bytecode), + upb_pbdecoder_getopname(op), + arg); +#endif + switch (op) { + /* Technically, we are losing data if we see a 32-bit varint that is not + * properly sign-extended. We could detect this and error about the data + * loss, but proto2 does not do this, so we pass. */ + PRIMITIVE_OP(INT32, varint, int32, int32_t, uint64_t) + PRIMITIVE_OP(INT64, varint, int64, int64_t, uint64_t) + PRIMITIVE_OP(UINT32, varint, uint32, uint32_t, uint64_t) + PRIMITIVE_OP(UINT64, varint, uint64, uint64_t, uint64_t) + PRIMITIVE_OP(FIXED32, fixed32, uint32, uint32_t, uint32_t) + PRIMITIVE_OP(FIXED64, fixed64, uint64, uint64_t, uint64_t) + PRIMITIVE_OP(SFIXED32, fixed32, int32, int32_t, uint32_t) + PRIMITIVE_OP(SFIXED64, fixed64, int64, int64_t, uint64_t) + PRIMITIVE_OP(BOOL, varint, bool, bool, uint64_t) + PRIMITIVE_OP(DOUBLE, fixed64, double, as_double, uint64_t) + PRIMITIVE_OP(FLOAT, fixed32, float, as_float, uint32_t) + PRIMITIVE_OP(SINT32, varint, int32, upb_zzdec_32, uint64_t) + PRIMITIVE_OP(SINT64, varint, int64, upb_zzdec_64, uint64_t) + + VMCASE(OP_SETDISPATCH, + d->top->base = d->pc - 1; + memcpy(&d->top->dispatch, d->pc, sizeof(void*)); + d->pc += sizeof(void*) / sizeof(uint32_t); + ) + VMCASE(OP_STARTMSG, + CHECK_SUSPEND(upb_sink_startmsg(d->top->sink)); + ) + VMCASE(OP_ENDMSG, + CHECK_SUSPEND(upb_sink_endmsg(d->top->sink, d->status)); + ) + VMCASE(OP_STARTSEQ, + upb_pbdecoder_frame *outer = outer_frame(d); + CHECK_SUSPEND(upb_sink_startseq(outer->sink, arg, &d->top->sink)); + ) + VMCASE(OP_ENDSEQ, + CHECK_SUSPEND(upb_sink_endseq(d->top->sink, arg)); + ) + VMCASE(OP_STARTSUBMSG, + upb_pbdecoder_frame *outer = outer_frame(d); + CHECK_SUSPEND(upb_sink_startsubmsg(outer->sink, arg, &d->top->sink)); + ) + VMCASE(OP_ENDSUBMSG, + CHECK_SUSPEND(upb_sink_endsubmsg(d->top->sink, arg)); + ) + VMCASE(OP_STARTSTR, + uint32_t len = delim_remaining(d); + upb_pbdecoder_frame *outer = outer_frame(d); + CHECK_SUSPEND(upb_sink_startstr(outer->sink, arg, len, &d->top->sink)); + if (len == 0) { + d->pc++; /* Skip OP_STRING. */ + } + ) + VMCASE(OP_STRING, + uint32_t len = curbufleft(d); + size_t n = upb_sink_putstring(d->top->sink, arg, d->ptr, len, handle); + if (n > len) { + if (n > delim_remaining(d)) { + seterr(d, "Tried to skip past end of string."); + return upb_pbdecoder_suspend(d); + } else { + int32_t ret = skip(d, n); + /* This shouldn't return DECODE_OK, because n > len. */ + UPB_ASSERT(ret >= 0); + return ret; + } + } + advance(d, n); + if (n < len || d->delim_end == NULL) { + /* We aren't finished with this string yet. */ + d->pc--; /* Repeat OP_STRING. */ + if (n > 0) checkpoint(d); + return upb_pbdecoder_suspend(d); + } + ) + VMCASE(OP_ENDSTR, + CHECK_SUSPEND(upb_sink_endstr(d->top->sink, arg)); + ) + VMCASE(OP_PUSHTAGDELIM, + CHECK_SUSPEND(pushtagdelim(d, arg)); + ) + VMCASE(OP_SETBIGGROUPNUM, + d->top->groupnum = *d->pc++; + ) + VMCASE(OP_POP, + UPB_ASSERT(d->top > d->stack); + decoder_pop(d); + ) + VMCASE(OP_PUSHLENDELIM, + uint32_t len; + CHECK_RETURN(decode_v32(d, &len)); + CHECK_SUSPEND(decoder_push(d, offset(d) + len)); + set_delim_end(d); + ) + VMCASE(OP_SETDELIM, + set_delim_end(d); + ) + VMCASE(OP_CHECKDELIM, + /* We are guaranteed of this assert because we never allow ourselves to + * consume bytes beyond data_end, which covers delim_end when non-NULL. + */ + UPB_ASSERT(!(d->delim_end && d->ptr > d->delim_end)); + if (d->ptr == d->delim_end) + d->pc += longofs; + ) + VMCASE(OP_CALL, + d->callstack[d->call_len++] = d->pc; + d->pc += longofs; + ) + VMCASE(OP_RET, + UPB_ASSERT(d->call_len > 0); + d->pc = d->callstack[--d->call_len]; + ) + VMCASE(OP_BRANCH, + d->pc += longofs; + ) + VMCASE(OP_TAG1, + uint8_t expected; + CHECK_SUSPEND(curbufleft(d) > 0); + expected = (arg >> 8) & 0xff; + if (*d->ptr == expected) { + advance(d, 1); + } else { + int8_t shortofs; + badtag: + shortofs = arg; + if (shortofs == LABEL_DISPATCH) { + CHECK_RETURN(dispatch(d)); + } else { + d->pc += shortofs; + break; /* Avoid checkpoint(). */ + } + } + ) + VMCASE(OP_TAG2, + uint16_t expected; + CHECK_SUSPEND(curbufleft(d) > 0); + expected = (arg >> 8) & 0xffff; + if (curbufleft(d) >= 2) { + uint16_t actual; + memcpy(&actual, d->ptr, 2); + if (expected == actual) { + advance(d, 2); + } else { + goto badtag; + } + } else { + int32_t result = upb_pbdecoder_checktag_slow(d, expected); + if (result == DECODE_MISMATCH) goto badtag; + if (result >= 0) return result; + } + ) + VMCASE(OP_TAGN, { + uint64_t expected; + int32_t result; + memcpy(&expected, d->pc, 8); + d->pc += 2; + result = upb_pbdecoder_checktag_slow(d, expected); + if (result == DECODE_MISMATCH) goto badtag; + if (result >= 0) return result; + }) + VMCASE(OP_DISPATCH, { + CHECK_RETURN(dispatch(d)); + }) + VMCASE(OP_HALT, { + return d->size_param; + }) + } + } +} + + +/* BytesHandler handlers ******************************************************/ + +void *upb_pbdecoder_startbc(void *closure, const void *pc, size_t size_hint) { + upb_pbdecoder *d = closure; + UPB_UNUSED(size_hint); + d->top->end_ofs = UINT64_MAX; + d->bufstart_ofs = 0; + d->call_len = 1; + d->callstack[0] = &halt; + d->pc = pc; + d->skip = 0; + return d; +} + +bool upb_pbdecoder_end(void *closure, const void *handler_data) { + upb_pbdecoder *d = closure; + const upb_pbdecodermethod *method = handler_data; + uint64_t end; + char dummy; + + if (d->residual_end > d->residual) { + seterr(d, "Unexpected EOF: decoder still has buffered unparsed data"); + return false; + } + + if (d->skip) { + seterr(d, "Unexpected EOF inside skipped data"); + return false; + } + + if (d->top->end_ofs != UINT64_MAX) { + seterr(d, "Unexpected EOF inside delimited string"); + return false; + } + + /* The user's end() call indicates that the message ends here. */ + end = offset(d); + d->top->end_ofs = end; + + { + const uint32_t *p = d->pc; + d->stack->end_ofs = end; + /* Check the previous bytecode, but guard against beginning. */ + if (p != method->code_base.ptr) p--; + if (getop(*p) == OP_CHECKDELIM) { + /* Rewind from OP_TAG* to OP_CHECKDELIM. */ + UPB_ASSERT(getop(*d->pc) == OP_TAG1 || + getop(*d->pc) == OP_TAG2 || + getop(*d->pc) == OP_TAGN || + getop(*d->pc) == OP_DISPATCH); + d->pc = p; + } + upb_pbdecoder_decode(closure, handler_data, &dummy, 0, NULL); + } + + if (d->call_len != 0) { + seterr(d, "Unexpected EOF inside submessage or group"); + return false; + } + + return true; +} + +size_t upb_pbdecoder_decode(void *decoder, const void *group, const char *buf, + size_t size, const upb_bufhandle *handle) { + int32_t result = upb_pbdecoder_resume(decoder, NULL, buf, size, handle); + + if (result == DECODE_ENDGROUP) goto_endmsg(decoder); + CHECK_RETURN(result); + + return run_decoder_vm(decoder, group, handle); +} + + +/* Public API *****************************************************************/ + +void upb_pbdecoder_reset(upb_pbdecoder *d) { + d->top = d->stack; + d->top->groupnum = 0; + d->ptr = d->residual; + d->buf = d->residual; + d->end = d->residual; + d->residual_end = d->residual; +} + +upb_pbdecoder *upb_pbdecoder_create(upb_arena *a, const upb_pbdecodermethod *m, + upb_sink sink, upb_status *status) { + const size_t default_max_nesting = 64; +#ifndef NDEBUG + size_t size_before = upb_arena_bytesallocated(a); +#endif + + upb_pbdecoder *d = upb_arena_malloc(a, sizeof(upb_pbdecoder)); + if (!d) return NULL; + + d->method_ = m; + d->callstack = upb_arena_malloc(a, callstacksize(d, default_max_nesting)); + d->stack = upb_arena_malloc(a, stacksize(d, default_max_nesting)); + if (!d->stack || !d->callstack) { + return NULL; + } + + d->arena = a; + d->limit = d->stack + default_max_nesting - 1; + d->stack_size = default_max_nesting; + d->status = status; + + upb_pbdecoder_reset(d); + upb_bytessink_reset(&d->input_, &m->input_handler_, d); + + if (d->method_->dest_handlers_) { + if (sink.handlers != d->method_->dest_handlers_) + return NULL; + } + d->top->sink = sink; + + /* If this fails, increase the value in decoder.h. */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(a) - size_before <= + UPB_PB_DECODER_SIZE); + return d; +} + +uint64_t upb_pbdecoder_bytesparsed(const upb_pbdecoder *d) { + return offset(d); +} + +const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d) { + return d->method_; +} + +upb_bytessink upb_pbdecoder_input(upb_pbdecoder *d) { + return d->input_; +} + +size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d) { + return d->stack_size; +} + +bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max) { + UPB_ASSERT(d->top >= d->stack); + + if (max < (size_t)(d->top - d->stack)) { + /* Can't set a limit smaller than what we are currently at. */ + return false; + } + + if (max > d->stack_size) { + /* Need to reallocate stack and callstack to accommodate. */ + size_t old_size = stacksize(d, d->stack_size); + size_t new_size = stacksize(d, max); + void *p = upb_arena_realloc(d->arena, d->stack, old_size, new_size); + if (!p) { + return false; + } + d->stack = p; + + old_size = callstacksize(d, d->stack_size); + new_size = callstacksize(d, max); + p = upb_arena_realloc(d->arena, d->callstack, old_size, new_size); + if (!p) { + return false; + } + d->callstack = p; + + d->stack_size = max; + } + + d->limit = d->stack + max - 1; + return true; +} diff --git a/third_party/upb/upb/pb/decoder.h b/third_party/upb/upb/pb/decoder.h new file mode 100644 index 00000000000..709db49e96a --- /dev/null +++ b/third_party/upb/upb/pb/decoder.h @@ -0,0 +1,240 @@ +/* +** upb::pb::Decoder +** +** A high performance, streaming, resumable decoder for the binary protobuf +** format. +** +** This interface works the same regardless of what decoder backend is being +** used. A client of this class does not need to know whether decoding is using +** a JITted decoder (DynASM, LLVM, etc) or an interpreted decoder. By default, +** it will always use the fastest available decoder. However, you can call +** set_allow_jit(false) to disable any JIT decoder that might be available. +** This is primarily useful for testing purposes. +*/ + +#ifndef UPB_DECODER_H_ +#define UPB_DECODER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace pb { +class CodeCache; +class DecoderPtr; +class DecoderMethodPtr; +class DecoderMethodOptions; +} /* namespace pb */ +} /* namespace upb */ +#endif + +/* The maximum number of bytes we are required to buffer internally between + * calls to the decoder. The value is 14: a 5 byte unknown tag plus ten-byte + * varint, less one because we are buffering an incomplete value. + * + * Should only be used by unit tests. */ +#define UPB_DECODER_MAX_RESIDUAL_BYTES 14 + +/* upb_pbdecodermethod ********************************************************/ + +struct upb_pbdecodermethod; +typedef struct upb_pbdecodermethod upb_pbdecodermethod; + +#ifdef __cplusplus +extern "C" { +#endif + +const upb_handlers *upb_pbdecodermethod_desthandlers( + const upb_pbdecodermethod *m); +const upb_byteshandler *upb_pbdecodermethod_inputhandler( + const upb_pbdecodermethod *m); +bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Represents the code to parse a protobuf according to a destination + * Handlers. */ +class upb::pb::DecoderMethodPtr { + public: + DecoderMethodPtr() : ptr_(nullptr) {} + DecoderMethodPtr(const upb_pbdecodermethod* ptr) : ptr_(ptr) {} + + const upb_pbdecodermethod* ptr() { return ptr_; } + + /* The destination handlers that are statically bound to this method. + * This method is only capable of outputting to a sink that uses these + * handlers. */ + const Handlers *dest_handlers() const { + return upb_pbdecodermethod_desthandlers(ptr_); + } + + /* The input handlers for this decoder method. */ + const BytesHandler* input_handler() const { + return upb_pbdecodermethod_inputhandler(ptr_); + } + + /* Whether this method is native. */ + bool is_native() const { + return upb_pbdecodermethod_isnative(ptr_); + } + + private: + const upb_pbdecodermethod* ptr_; +}; + +#endif + +/* upb_pbdecoder **************************************************************/ + +/* Preallocation hint: decoder won't allocate more bytes than this when first + * constructed. This hint may be an overestimate for some build configurations. + * But if the decoder library is upgraded without recompiling the application, + * it may be an underestimate. */ +#define UPB_PB_DECODER_SIZE 4416 + +struct upb_pbdecoder; +typedef struct upb_pbdecoder upb_pbdecoder; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_pbdecoder *upb_pbdecoder_create(upb_arena *arena, + const upb_pbdecodermethod *method, + upb_sink output, upb_status *status); +const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d); +upb_bytessink upb_pbdecoder_input(upb_pbdecoder *d); +uint64_t upb_pbdecoder_bytesparsed(const upb_pbdecoder *d); +size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d); +bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max); +void upb_pbdecoder_reset(upb_pbdecoder *d); + +#ifdef __cplusplus +} /* extern "C" */ + +/* A Decoder receives binary protobuf data on its input sink and pushes the + * decoded data to its output sink. */ +class upb::pb::DecoderPtr { + public: + DecoderPtr() : ptr_(nullptr) {} + DecoderPtr(upb_pbdecoder* ptr) : ptr_(ptr) {} + + upb_pbdecoder* ptr() { return ptr_; } + + /* Constructs a decoder instance for the given method, which must outlive this + * decoder. Any errors during parsing will be set on the given status, which + * must also outlive this decoder. + * + * The sink must match the given method. */ + static DecoderPtr Create(Arena *arena, DecoderMethodPtr method, + upb::Sink output, Status *status) { + return DecoderPtr(upb_pbdecoder_create(arena->ptr(), method.ptr(), + output.sink(), status->ptr())); + } + + /* Returns the DecoderMethod this decoder is parsing from. */ + const DecoderMethodPtr method() const { + return DecoderMethodPtr(upb_pbdecoder_method(ptr_)); + } + + /* The sink on which this decoder receives input. */ + BytesSink input() { return BytesSink(upb_pbdecoder_input(ptr())); } + + /* Returns number of bytes successfully parsed. + * + * This can be useful for determining the stream position where an error + * occurred. + * + * This value may not be up-to-date when called from inside a parsing + * callback. */ + uint64_t BytesParsed() { return upb_pbdecoder_bytesparsed(ptr()); } + + /* Gets/sets the parsing nexting limit. If the total number of nested + * submessages and repeated fields hits this limit, parsing will fail. This + * is a resource limit that controls the amount of memory used by the parsing + * stack. + * + * Setting the limit will fail if the parser is currently suspended at a depth + * greater than this, or if memory allocation of the stack fails. */ + size_t max_nesting() { return upb_pbdecoder_maxnesting(ptr()); } + bool set_max_nesting(size_t max) { return upb_pbdecoder_maxnesting(ptr()); } + + void Reset() { upb_pbdecoder_reset(ptr()); } + + static const size_t kSize = UPB_PB_DECODER_SIZE; + + private: + upb_pbdecoder *ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_pbcodecache ************************************************************/ + +/* Lazily builds and caches decoder methods that will push data to the given + * handlers. The destination handlercache must outlive this object. */ + +struct upb_pbcodecache; +typedef struct upb_pbcodecache upb_pbcodecache; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_pbcodecache *upb_pbcodecache_new(upb_handlercache *dest); +void upb_pbcodecache_free(upb_pbcodecache *c); +bool upb_pbcodecache_allowjit(const upb_pbcodecache *c); +void upb_pbcodecache_setallowjit(upb_pbcodecache *c, bool allow); +void upb_pbcodecache_setlazy(upb_pbcodecache *c, bool lazy); +const upb_pbdecodermethod *upb_pbcodecache_get(upb_pbcodecache *c, + const upb_msgdef *md); + +#ifdef __cplusplus +} /* extern "C" */ + +/* A class for caching protobuf processing code, whether bytecode for the + * interpreted decoder or machine code for the JIT. + * + * This class is not thread-safe. */ +class upb::pb::CodeCache { + public: + CodeCache(upb::HandlerCache *dest) + : ptr_(upb_pbcodecache_new(dest->ptr()), upb_pbcodecache_free) {} + CodeCache(CodeCache&&) = default; + CodeCache& operator=(CodeCache&&) = default; + + upb_pbcodecache* ptr() { return ptr_.get(); } + const upb_pbcodecache* ptr() const { return ptr_.get(); } + + /* Whether the cache is allowed to generate machine code. Defaults to true. + * There is no real reason to turn it off except for testing or if you are + * having a specific problem with the JIT. + * + * Note that allow_jit = true does not *guarantee* that the code will be JIT + * compiled. If this platform is not supported or the JIT was not compiled + * in, the code may still be interpreted. */ + bool allow_jit() const { return upb_pbcodecache_allowjit(ptr()); } + + /* This may only be called when the object is first constructed, and prior to + * any code generation. */ + void set_allow_jit(bool allow) { upb_pbcodecache_setallowjit(ptr(), allow); } + + /* Should the decoder push submessages to lazy handlers for fields that have + * them? The caller should set this iff the lazy handlers expect data that is + * in protobuf binary format and the caller wishes to lazy parse it. */ + void set_lazy(bool lazy) { upb_pbcodecache_setlazy(ptr(), lazy); } + + /* Returns a DecoderMethod that can push data to the given handlers. + * If a suitable method already exists, it will be returned from the cache. */ + const DecoderMethodPtr Get(MessageDefPtr md) { + return DecoderMethodPtr(upb_pbcodecache_get(ptr(), md.ptr())); + } + + private: + std::unique_ptr ptr_; +}; + +#endif /* __cplusplus */ + +#endif /* UPB_DECODER_H_ */ diff --git a/third_party/upb/upb/pb/decoder.int.h b/third_party/upb/upb/pb/decoder.int.h new file mode 100644 index 00000000000..9d5f5839bc8 --- /dev/null +++ b/third_party/upb/upb/pb/decoder.int.h @@ -0,0 +1,288 @@ +/* +** Internal-only definitions for the decoder. +*/ + +#ifndef UPB_DECODER_INT_H_ +#define UPB_DECODER_INT_H_ + +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/sink.h" +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +/* Opcode definitions. The canonical meaning of each opcode is its + * implementation in the interpreter (the JIT is written to match this). + * + * All instructions have the opcode in the low byte. + * Instruction format for most instructions is: + * + * +-------------------+--------+ + * | arg (24) | op (8) | + * +-------------------+--------+ + * + * Exceptions are indicated below. A few opcodes are multi-word. */ +typedef enum { + /* Opcodes 1-8, 13, 15-18 parse their respective descriptor types. + * Arg for all of these is the upb selector for this field. */ +#define T(type) OP_PARSE_ ## type = UPB_DESCRIPTOR_TYPE_ ## type + T(DOUBLE), T(FLOAT), T(INT64), T(UINT64), T(INT32), T(FIXED64), T(FIXED32), + T(BOOL), T(UINT32), T(SFIXED32), T(SFIXED64), T(SINT32), T(SINT64), +#undef T + OP_STARTMSG = 9, /* No arg. */ + OP_ENDMSG = 10, /* No arg. */ + OP_STARTSEQ = 11, + OP_ENDSEQ = 12, + OP_STARTSUBMSG = 14, + OP_ENDSUBMSG = 19, + OP_STARTSTR = 20, + OP_STRING = 21, + OP_ENDSTR = 22, + + OP_PUSHTAGDELIM = 23, /* No arg. */ + OP_PUSHLENDELIM = 24, /* No arg. */ + OP_POP = 25, /* No arg. */ + OP_SETDELIM = 26, /* No arg. */ + OP_SETBIGGROUPNUM = 27, /* two words: + * | unused (24) | opc (8) | + * | groupnum (32) | */ + OP_CHECKDELIM = 28, + OP_CALL = 29, + OP_RET = 30, + OP_BRANCH = 31, + + /* Different opcodes depending on how many bytes expected. */ + OP_TAG1 = 32, /* | match tag (16) | jump target (8) | opc (8) | */ + OP_TAG2 = 33, /* | match tag (16) | jump target (8) | opc (8) | */ + OP_TAGN = 34, /* three words: */ + /* | unused (16) | jump target(8) | opc (8) | */ + /* | match tag 1 (32) | */ + /* | match tag 2 (32) | */ + + OP_SETDISPATCH = 35, /* N words: */ + /* | unused (24) | opc | */ + /* | upb_inttable* (32 or 64) | */ + + OP_DISPATCH = 36, /* No arg. */ + + OP_HALT = 37 /* No arg. */ +} opcode; + +#define OP_MAX OP_HALT + +UPB_INLINE opcode getop(uint32_t instr) { return (opcode)(instr & 0xff); } + +struct upb_pbcodecache { + upb_arena *arena; + upb_handlercache *dest; + bool allow_jit; + bool lazy; + + /* Map of upb_msgdef -> mgroup. */ + upb_inttable groups; +}; + +/* Method group; represents a set of decoder methods that had their code + * emitted together. Immutable once created. */ +typedef struct { + /* Maps upb_msgdef/upb_handlers -> upb_pbdecodermethod. Owned by us. + * + * Ideally this would be on pbcodecache (if we were actually caching code). + * Right now we don't actually cache anything, which is wasteful. */ + upb_inttable methods; + + /* The bytecode for our methods, if any exists. Owned by us. */ + uint32_t *bytecode; + uint32_t *bytecode_end; +} mgroup; + +/* The maximum that any submessages can be nested. Matches proto2's limit. + * This specifies the size of the decoder's statically-sized array and therefore + * setting it high will cause the upb::pb::Decoder object to be larger. + * + * If necessary we can add a runtime-settable property to Decoder that allow + * this to be larger than the compile-time setting, but this would add + * complexity, particularly since we would have to decide how/if to give users + * the ability to set a custom memory allocation function. */ +#define UPB_DECODER_MAX_NESTING 64 + +/* Internal-only struct used by the decoder. */ +typedef struct { + /* Space optimization note: we store two pointers here that the JIT + * doesn't need at all; the upb_handlers* inside the sink and + * the dispatch table pointer. We can optimze so that the JIT uses + * smaller stack frames than the interpreter. The only thing we need + * to guarantee is that the fallback routines can find end_ofs. */ + upb_sink sink; + + /* The absolute stream offset of the end-of-frame delimiter. + * Non-delimited frames (groups and non-packed repeated fields) reuse the + * delimiter of their parent, even though the frame may not end there. + * + * NOTE: the JIT stores a slightly different value here for non-top frames. + * It stores the value relative to the end of the enclosed message. But the + * top frame is still stored the same way, which is important for ensuring + * that calls from the JIT into C work correctly. */ + uint64_t end_ofs; + const uint32_t *base; + + /* 0 indicates a length-delimited field. + * A positive number indicates a known group. + * A negative number indicates an unknown group. */ + int32_t groupnum; + upb_inttable *dispatch; /* Not used by the JIT. */ +} upb_pbdecoder_frame; + +struct upb_pbdecodermethod { + /* While compiling, the base is relative in "ofs", after compiling it is + * absolute in "ptr". */ + union { + uint32_t ofs; /* PC offset of method. */ + void *ptr; /* Pointer to bytecode or machine code for this method. */ + } code_base; + + /* The decoder method group to which this method belongs. */ + const mgroup *group; + + /* Whether this method is native code or bytecode. */ + bool is_native_; + + /* The handler one calls to invoke this method. */ + upb_byteshandler input_handler_; + + /* The destination handlers this method is bound to. We own a ref. */ + const upb_handlers *dest_handlers_; + + /* Dispatch table -- used by both bytecode decoder and JIT when encountering a + * field number that wasn't the one we were expecting to see. See + * decoder.int.h for the layout of this table. */ + upb_inttable dispatch; +}; + +struct upb_pbdecoder { + upb_arena *arena; + + /* Our input sink. */ + upb_bytessink input_; + + /* The decoder method we are parsing with (owned). */ + const upb_pbdecodermethod *method_; + + size_t call_len; + const uint32_t *pc, *last; + + /* Current input buffer and its stream offset. */ + const char *buf, *ptr, *end, *checkpoint; + + /* End of the delimited region, relative to ptr, NULL if not in this buf. */ + const char *delim_end; + + /* End of the delimited region, relative to ptr, end if not in this buf. */ + const char *data_end; + + /* Overall stream offset of "buf." */ + uint64_t bufstart_ofs; + + /* Buffer for residual bytes not parsed from the previous buffer. */ + char residual[UPB_DECODER_MAX_RESIDUAL_BYTES]; + char *residual_end; + + /* Bytes of data that should be discarded from the input beore we start + * parsing again. We set this when we internally determine that we can + * safely skip the next N bytes, but this region extends past the current + * user buffer. */ + size_t skip; + + /* Stores the user buffer passed to our decode function. */ + const char *buf_param; + size_t size_param; + const upb_bufhandle *handle; + + /* Our internal stack. */ + upb_pbdecoder_frame *stack, *top, *limit; + const uint32_t **callstack; + size_t stack_size; + + upb_status *status; +}; + +/* Decoder entry points; used as handlers. */ +void *upb_pbdecoder_startbc(void *closure, const void *pc, size_t size_hint); +size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf, + size_t size, const upb_bufhandle *handle); +bool upb_pbdecoder_end(void *closure, const void *handler_data); + +/* Decoder-internal functions that the JIT calls to handle fallback paths. */ +int32_t upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf, + size_t size, const upb_bufhandle *handle); +size_t upb_pbdecoder_suspend(upb_pbdecoder *d); +int32_t upb_pbdecoder_skipunknown(upb_pbdecoder *d, int32_t fieldnum, + uint8_t wire_type); +int32_t upb_pbdecoder_checktag_slow(upb_pbdecoder *d, uint64_t expected); +int32_t upb_pbdecoder_decode_varint_slow(upb_pbdecoder *d, uint64_t *u64); +int32_t upb_pbdecoder_decode_f32(upb_pbdecoder *d, uint32_t *u32); +int32_t upb_pbdecoder_decode_f64(upb_pbdecoder *d, uint64_t *u64); +void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg); + +/* Error messages that are shared between the bytecode and JIT decoders. */ +extern const char *kPbDecoderStackOverflow; +extern const char *kPbDecoderSubmessageTooLong; + +/* Access to decoderplan members needed by the decoder. */ +const char *upb_pbdecoder_getopname(unsigned int op); + +/* A special label that means "do field dispatch for this message and branch to + * wherever that takes you." */ +#define LABEL_DISPATCH 0 + +/* A special slot in the dispatch table that stores the epilogue (ENDMSG and/or + * RET) for branching to when we find an appropriate ENDGROUP tag. */ +#define DISPATCH_ENDMSG 0 + +/* It's important to use this invalid wire type instead of 0 (which is a valid + * wire type). */ +#define NO_WIRE_TYPE 0xff + +/* The dispatch table layout is: + * [field number] -> [ 48-bit offset ][ 8-bit wt2 ][ 8-bit wt1 ] + * + * If wt1 matches, jump to the 48-bit offset. If wt2 matches, lookup + * (UPB_MAX_FIELDNUMBER + fieldnum) and jump there. + * + * We need two wire types because of packed/non-packed compatibility. A + * primitive repeated field can use either wire type and be valid. While we + * could key the table on fieldnum+wiretype, the table would be 8x sparser. + * + * Storing two wire types in the primary value allows us to quickly rule out + * the second wire type without needing to do a separate lookup (this case is + * less common than an unknown field). */ +UPB_INLINE uint64_t upb_pbdecoder_packdispatch(uint64_t ofs, uint8_t wt1, + uint8_t wt2) { + return (ofs << 16) | (wt2 << 8) | wt1; +} + +UPB_INLINE void upb_pbdecoder_unpackdispatch(uint64_t dispatch, uint64_t *ofs, + uint8_t *wt1, uint8_t *wt2) { + *wt1 = (uint8_t)dispatch; + *wt2 = (uint8_t)(dispatch >> 8); + *ofs = dispatch >> 16; +} + +/* All of the functions in decoder.c that return int32_t return values according + * to the following scheme: + * 1. negative values indicate a return code from the following list. + * 2. positive values indicate that error or end of buffer was hit, and + * that the decode function should immediately return the given value + * (the decoder state has already been suspended and is ready to be + * resumed). */ +#define DECODE_OK -1 +#define DECODE_MISMATCH -2 /* Used only from checktag_slow(). */ +#define DECODE_ENDGROUP -3 /* Used only from checkunknown(). */ + +#define CHECK_RETURN(x) { int32_t ret = x; if (ret >= 0) return ret; } + +#include "upb/port_undef.inc" + +#endif /* UPB_DECODER_INT_H_ */ diff --git a/third_party/upb/upb/pb/encoder.c b/third_party/upb/upb/pb/encoder.c new file mode 100644 index 00000000000..d108a643d30 --- /dev/null +++ b/third_party/upb/upb/pb/encoder.c @@ -0,0 +1,570 @@ +/* +** upb::Encoder +** +** Since we are implementing pure handlers (ie. without any out-of-band access +** to pre-computed lengths), we have to buffer all submessages before we can +** emit even their first byte. +** +** Not knowing the size of submessages also means we can't write a perfect +** zero-copy implementation, even with buffering. Lengths are stored as +** varints, which means that we don't know how many bytes to reserve for the +** length until we know what the length is. +** +** This leaves us with three main choices: +** +** 1. buffer all submessage data in a temporary buffer, then copy it exactly +** once into the output buffer. +** +** 2. attempt to buffer data directly into the output buffer, estimating how +** many bytes each length will take. When our guesses are wrong, use +** memmove() to grow or shrink the allotted space. +** +** 3. buffer directly into the output buffer, allocating a max length +** ahead-of-time for each submessage length. If we overallocated, we waste +** space, but no memcpy() or memmove() is required. This approach requires +** defining a maximum size for submessages and rejecting submessages that +** exceed that size. +** +** (2) and (3) have the potential to have better performance, but they are more +** complicated and subtle to implement: +** +** (3) requires making an arbitrary choice of the maximum message size; it +** wastes space when submessages are shorter than this and fails +** completely when they are longer. This makes it more finicky and +** requires configuration based on the input. It also makes it impossible +** to perfectly match the output of reference encoders that always use the +** optimal amount of space for each length. +** +** (2) requires guessing the the size upfront, and if multiple lengths are +** guessed wrong the minimum required number of memmove() operations may +** be complicated to compute correctly. Implemented properly, it may have +** a useful amortized or average cost, but more investigation is required +** to determine this and what the optimal algorithm is to achieve it. +** +** (1) makes you always pay for exactly one copy, but its implementation is +** the simplest and its performance is predictable. +** +** So for now, we implement (1) only. If we wish to optimize later, we should +** be able to do it without affecting users. +** +** The strategy is to buffer the segments of data that do *not* depend on +** unknown lengths in one buffer, and keep a separate buffer of segment pointers +** and lengths. When the top-level submessage ends, we can go beginning to end, +** alternating the writing of lengths with memcpy() of the rest of the data. +** At the top level though, no buffering is required. +*/ + +#include "upb/pb/encoder.h" +#include "upb/pb/varint.int.h" + +#include "upb/port_def.inc" + +/* The output buffer is divided into segments; a segment is a string of data + * that is "ready to go" -- it does not need any varint lengths inserted into + * the middle. The seams between segments are where varints will be inserted + * once they are known. + * + * We also use the concept of a "run", which is a range of encoded bytes that + * occur at a single submessage level. Every segment contains one or more runs. + * + * A segment can span messages. Consider: + * + * .--Submessage lengths---------. + * | | | + * | V V + * V | |--------------- | |----------------- + * Submessages: | |----------------------------------------------- + * Top-level msg: ------------------------------------------------------------ + * + * Segments: ----- ------------------- ----------------- + * Runs: *---- *--------------*--- *---------------- + * (* marks the start) + * + * Note that the top-level menssage is not in any segment because it does not + * have any length preceding it. + * + * A segment is only interrupted when another length needs to be inserted. So + * observe how the second segment spans both the inner submessage and part of + * the next enclosing message. */ +typedef struct { + uint32_t msglen; /* The length to varint-encode before this segment. */ + uint32_t seglen; /* Length of the segment. */ +} upb_pb_encoder_segment; + +struct upb_pb_encoder { + upb_arena *arena; + + /* Our input and output. */ + upb_sink input_; + upb_bytessink output_; + + /* The "subclosure" -- used as the inner closure as part of the bytessink + * protocol. */ + void *subc; + + /* The output buffer and limit, and our current write position. "buf" + * initially points to "initbuf", but is dynamically allocated if we need to + * grow beyond the initial size. */ + char *buf, *ptr, *limit; + + /* The beginning of the current run, or undefined if we are at the top + * level. */ + char *runbegin; + + /* The list of segments we are accumulating. */ + upb_pb_encoder_segment *segbuf, *segptr, *seglimit; + + /* The stack of enclosing submessages. Each entry in the stack points to the + * segment where this submessage's length is being accumulated. */ + int *stack, *top, *stacklimit; + + /* Depth of startmsg/endmsg calls. */ + int depth; +}; + +/* low-level buffering ********************************************************/ + +/* Low-level functions for interacting with the output buffer. */ + +/* TODO(haberman): handle pushback */ +static void putbuf(upb_pb_encoder *e, const char *buf, size_t len) { + size_t n = upb_bytessink_putbuf(e->output_, e->subc, buf, len, NULL); + UPB_ASSERT(n == len); +} + +static upb_pb_encoder_segment *top(upb_pb_encoder *e) { + return &e->segbuf[*e->top]; +} + +/* Call to ensure that at least "bytes" bytes are available for writing at + * e->ptr. Returns false if the bytes could not be allocated. */ +static bool reserve(upb_pb_encoder *e, size_t bytes) { + if ((size_t)(e->limit - e->ptr) < bytes) { + /* Grow buffer. */ + char *new_buf; + size_t needed = bytes + (e->ptr - e->buf); + size_t old_size = e->limit - e->buf; + + size_t new_size = old_size; + + while (new_size < needed) { + new_size *= 2; + } + + new_buf = upb_arena_realloc(e->arena, e->buf, old_size, new_size); + + if (new_buf == NULL) { + return false; + } + + e->ptr = new_buf + (e->ptr - e->buf); + e->runbegin = new_buf + (e->runbegin - e->buf); + e->limit = new_buf + new_size; + e->buf = new_buf; + } + + return true; +} + +/* Call when "bytes" bytes have been writte at e->ptr. The caller *must* have + * previously called reserve() with at least this many bytes. */ +static void encoder_advance(upb_pb_encoder *e, size_t bytes) { + UPB_ASSERT((size_t)(e->limit - e->ptr) >= bytes); + e->ptr += bytes; +} + +/* Call when all of the bytes for a handler have been written. Flushes the + * bytes if possible and necessary, returning false if this failed. */ +static bool commit(upb_pb_encoder *e) { + if (!e->top) { + /* We aren't inside a delimited region. Flush our accumulated bytes to + * the output. + * + * TODO(haberman): in the future we may want to delay flushing for + * efficiency reasons. */ + putbuf(e, e->buf, e->ptr - e->buf); + e->ptr = e->buf; + } + + return true; +} + +/* Writes the given bytes to the buffer, handling reserve/advance. */ +static bool encode_bytes(upb_pb_encoder *e, const void *data, size_t len) { + if (!reserve(e, len)) { + return false; + } + + memcpy(e->ptr, data, len); + encoder_advance(e, len); + return true; +} + +/* Finish the current run by adding the run totals to the segment and message + * length. */ +static void accumulate(upb_pb_encoder *e) { + size_t run_len; + UPB_ASSERT(e->ptr >= e->runbegin); + run_len = e->ptr - e->runbegin; + e->segptr->seglen += run_len; + top(e)->msglen += run_len; + e->runbegin = e->ptr; +} + +/* Call to indicate the start of delimited region for which the full length is + * not yet known. All data will be buffered until the length is known. + * Delimited regions may be nested; their lengths will all be tracked properly. */ +static bool start_delim(upb_pb_encoder *e) { + if (e->top) { + /* We are already buffering, advance to the next segment and push it on the + * stack. */ + accumulate(e); + + if (++e->top == e->stacklimit) { + /* TODO(haberman): grow stack? */ + return false; + } + + if (++e->segptr == e->seglimit) { + /* Grow segment buffer. */ + size_t old_size = + (e->seglimit - e->segbuf) * sizeof(upb_pb_encoder_segment); + size_t new_size = old_size * 2; + upb_pb_encoder_segment *new_buf = + upb_arena_realloc(e->arena, e->segbuf, old_size, new_size); + + if (new_buf == NULL) { + return false; + } + + e->segptr = new_buf + (e->segptr - e->segbuf); + e->seglimit = new_buf + (new_size / sizeof(upb_pb_encoder_segment)); + e->segbuf = new_buf; + } + } else { + /* We were previously at the top level, start buffering. */ + e->segptr = e->segbuf; + e->top = e->stack; + e->runbegin = e->ptr; + } + + *e->top = e->segptr - e->segbuf; + e->segptr->seglen = 0; + e->segptr->msglen = 0; + + return true; +} + +/* Call to indicate the end of a delimited region. We now know the length of + * the delimited region. If we are not nested inside any other delimited + * regions, we can now emit all of the buffered data we accumulated. */ +static bool end_delim(upb_pb_encoder *e) { + size_t msglen; + accumulate(e); + msglen = top(e)->msglen; + + if (e->top == e->stack) { + /* All lengths are now available, emit all buffered data. */ + char buf[UPB_PB_VARINT_MAX_LEN]; + upb_pb_encoder_segment *s; + const char *ptr = e->buf; + for (s = e->segbuf; s <= e->segptr; s++) { + size_t lenbytes = upb_vencode64(s->msglen, buf); + putbuf(e, buf, lenbytes); + putbuf(e, ptr, s->seglen); + ptr += s->seglen; + } + + e->ptr = e->buf; + e->top = NULL; + } else { + /* Need to keep buffering; propagate length info into enclosing + * submessages. */ + --e->top; + top(e)->msglen += msglen + upb_varint_size(msglen); + } + + return true; +} + + +/* tag_t **********************************************************************/ + +/* A precomputed (pre-encoded) tag and length. */ + +typedef struct { + uint8_t bytes; + char tag[7]; +} tag_t; + +/* Allocates a new tag for this field, and sets it in these handlerattr. */ +static void new_tag(upb_handlers *h, const upb_fielddef *f, upb_wiretype_t wt, + upb_handlerattr *attr) { + uint32_t n = upb_fielddef_number(f); + + tag_t *tag = upb_gmalloc(sizeof(tag_t)); + tag->bytes = upb_vencode64((n << 3) | wt, tag->tag); + + attr->handler_data = tag; + upb_handlers_addcleanup(h, tag, upb_gfree); +} + +static bool encode_tag(upb_pb_encoder *e, const tag_t *tag) { + return encode_bytes(e, tag->tag, tag->bytes); +} + + +/* encoding of wire types *****************************************************/ + +static bool encode_fixed64(upb_pb_encoder *e, uint64_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return encode_bytes(e, &val, sizeof(uint64_t)); +} + +static bool encode_fixed32(upb_pb_encoder *e, uint32_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return encode_bytes(e, &val, sizeof(uint32_t)); +} + +static bool encode_varint(upb_pb_encoder *e, uint64_t val) { + if (!reserve(e, UPB_PB_VARINT_MAX_LEN)) { + return false; + } + + encoder_advance(e, upb_vencode64(val, e->ptr)); + return true; +} + +static uint64_t dbl2uint64(double d) { + uint64_t ret; + memcpy(&ret, &d, sizeof(uint64_t)); + return ret; +} + +static uint32_t flt2uint32(float d) { + uint32_t ret; + memcpy(&ret, &d, sizeof(uint32_t)); + return ret; +} + + +/* encoding of proto types ****************************************************/ + +static bool startmsg(void *c, const void *hd) { + upb_pb_encoder *e = c; + UPB_UNUSED(hd); + if (e->depth++ == 0) { + upb_bytessink_start(e->output_, 0, &e->subc); + } + return true; +} + +static bool endmsg(void *c, const void *hd, upb_status *status) { + upb_pb_encoder *e = c; + UPB_UNUSED(hd); + UPB_UNUSED(status); + if (--e->depth == 0) { + upb_bytessink_end(e->output_); + } + return true; +} + +static void *encode_startdelimfield(void *c, const void *hd) { + bool ok = encode_tag(c, hd) && commit(c) && start_delim(c); + return ok ? c : UPB_BREAK; +} + +static bool encode_unknown(void *c, const void *hd, const char *buf, + size_t len) { + UPB_UNUSED(hd); + return encode_bytes(c, buf, len) && commit(c); +} + +static bool encode_enddelimfield(void *c, const void *hd) { + UPB_UNUSED(hd); + return end_delim(c); +} + +static void *encode_startgroup(void *c, const void *hd) { + return (encode_tag(c, hd) && commit(c)) ? c : UPB_BREAK; +} + +static bool encode_endgroup(void *c, const void *hd) { + return encode_tag(c, hd) && commit(c); +} + +static void *encode_startstr(void *c, const void *hd, size_t size_hint) { + UPB_UNUSED(size_hint); + return encode_startdelimfield(c, hd); +} + +static size_t encode_strbuf(void *c, const void *hd, const char *buf, + size_t len, const upb_bufhandle *h) { + UPB_UNUSED(hd); + UPB_UNUSED(h); + return encode_bytes(c, buf, len) ? len : 0; +} + +#define T(type, ctype, convert, encode) \ + static bool encode_scalar_##type(void *e, const void *hd, ctype val) { \ + return encode_tag(e, hd) && encode(e, (convert)(val)) && commit(e); \ + } \ + static bool encode_packed_##type(void *e, const void *hd, ctype val) { \ + UPB_UNUSED(hd); \ + return encode(e, (convert)(val)); \ + } + +T(double, double, dbl2uint64, encode_fixed64) +T(float, float, flt2uint32, encode_fixed32) +T(int64, int64_t, uint64_t, encode_varint) +T(int32, int32_t, int64_t, encode_varint) +T(fixed64, uint64_t, uint64_t, encode_fixed64) +T(fixed32, uint32_t, uint32_t, encode_fixed32) +T(bool, bool, bool, encode_varint) +T(uint32, uint32_t, uint32_t, encode_varint) +T(uint64, uint64_t, uint64_t, encode_varint) +T(enum, int32_t, uint32_t, encode_varint) +T(sfixed32, int32_t, uint32_t, encode_fixed32) +T(sfixed64, int64_t, uint64_t, encode_fixed64) +T(sint32, int32_t, upb_zzenc_32, encode_varint) +T(sint64, int64_t, upb_zzenc_64, encode_varint) + +#undef T + + +/* code to build the handlers *************************************************/ + +#include +static void newhandlers_callback(const void *closure, upb_handlers *h) { + const upb_msgdef *m; + upb_msg_field_iter i; + + UPB_UNUSED(closure); + + upb_handlers_setstartmsg(h, startmsg, NULL); + upb_handlers_setendmsg(h, endmsg, NULL); + upb_handlers_setunknown(h, encode_unknown, NULL); + + m = upb_handlers_msgdef(h); + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + bool packed = upb_fielddef_isseq(f) && upb_fielddef_isprimitive(f) && + upb_fielddef_packed(f); + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + upb_wiretype_t wt = + packed ? UPB_WIRE_TYPE_DELIMITED + : upb_pb_native_wire_types[upb_fielddef_descriptortype(f)]; + + /* Pre-encode the tag for this field. */ + new_tag(h, f, wt, &attr); + + if (packed) { + upb_handlers_setstartseq(h, f, encode_startdelimfield, &attr); + upb_handlers_setendseq(h, f, encode_enddelimfield, &attr); + } + +#define T(upper, lower, upbtype) \ + case UPB_DESCRIPTOR_TYPE_##upper: \ + if (packed) { \ + upb_handlers_set##upbtype(h, f, encode_packed_##lower, &attr); \ + } else { \ + upb_handlers_set##upbtype(h, f, encode_scalar_##lower, &attr); \ + } \ + break; + + switch (upb_fielddef_descriptortype(f)) { + T(DOUBLE, double, double); + T(FLOAT, float, float); + T(INT64, int64, int64); + T(INT32, int32, int32); + T(FIXED64, fixed64, uint64); + T(FIXED32, fixed32, uint32); + T(BOOL, bool, bool); + T(UINT32, uint32, uint32); + T(UINT64, uint64, uint64); + T(ENUM, enum, int32); + T(SFIXED32, sfixed32, int32); + T(SFIXED64, sfixed64, int64); + T(SINT32, sint32, int32); + T(SINT64, sint64, int64); + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: + upb_handlers_setstartstr(h, f, encode_startstr, &attr); + upb_handlers_setendstr(h, f, encode_enddelimfield, &attr); + upb_handlers_setstring(h, f, encode_strbuf, &attr); + break; + case UPB_DESCRIPTOR_TYPE_MESSAGE: + upb_handlers_setstartsubmsg(h, f, encode_startdelimfield, &attr); + upb_handlers_setendsubmsg(h, f, encode_enddelimfield, &attr); + break; + case UPB_DESCRIPTOR_TYPE_GROUP: { + /* Endgroup takes a different tag (wire_type = END_GROUP). */ + upb_handlerattr attr2 = UPB_HANDLERATTR_INIT; + new_tag(h, f, UPB_WIRE_TYPE_END_GROUP, &attr2); + + upb_handlers_setstartsubmsg(h, f, encode_startgroup, &attr); + upb_handlers_setendsubmsg(h, f, encode_endgroup, &attr2); + + break; + } + } + +#undef T + } +} + +void upb_pb_encoder_reset(upb_pb_encoder *e) { + e->segptr = NULL; + e->top = NULL; + e->depth = 0; +} + + +/* public API *****************************************************************/ + +upb_handlercache *upb_pb_encoder_newcache(void) { + return upb_handlercache_new(newhandlers_callback, NULL); +} + +upb_pb_encoder *upb_pb_encoder_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output) { + const size_t initial_bufsize = 256; + const size_t initial_segbufsize = 16; + /* TODO(haberman): make this configurable. */ + const size_t stack_size = 64; +#ifndef NDEBUG + const size_t size_before = upb_arena_bytesallocated(arena); +#endif + + upb_pb_encoder *e = upb_arena_malloc(arena, sizeof(upb_pb_encoder)); + if (!e) return NULL; + + e->buf = upb_arena_malloc(arena, initial_bufsize); + e->segbuf = upb_arena_malloc(arena, initial_segbufsize * sizeof(*e->segbuf)); + e->stack = upb_arena_malloc(arena, stack_size * sizeof(*e->stack)); + + if (!e->buf || !e->segbuf || !e->stack) { + return NULL; + } + + e->limit = e->buf + initial_bufsize; + e->seglimit = e->segbuf + initial_segbufsize; + e->stacklimit = e->stack + stack_size; + + upb_pb_encoder_reset(e); + upb_sink_reset(&e->input_, h, e); + + e->arena = arena; + e->output_ = output; + e->subc = output.closure; + e->ptr = e->buf; + + /* If this fails, increase the value in encoder.h. */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= + UPB_PB_ENCODER_SIZE); + return e; +} + +upb_sink upb_pb_encoder_input(upb_pb_encoder *e) { return e->input_; } diff --git a/third_party/upb/upb/pb/encoder.h b/third_party/upb/upb/pb/encoder.h new file mode 100644 index 00000000000..f125b372188 --- /dev/null +++ b/third_party/upb/upb/pb/encoder.h @@ -0,0 +1,83 @@ +/* +** upb::pb::Encoder (upb_pb_encoder) +** +** Implements a set of upb_handlers that write protobuf data to the binary wire +** format. +** +** This encoder implementation does not have any access to any out-of-band or +** precomputed lengths for submessages, so it must buffer submessages internally +** before it can emit the first byte. +*/ + +#ifndef UPB_ENCODER_H_ +#define UPB_ENCODER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace pb { +class EncoderPtr; +} /* namespace pb */ +} /* namespace upb */ +#endif + +#define UPB_PBENCODER_MAX_NESTING 100 + +/* upb_pb_encoder *************************************************************/ + +/* Preallocation hint: decoder won't allocate more bytes than this when first + * constructed. This hint may be an overestimate for some build configurations. + * But if the decoder library is upgraded without recompiling the application, + * it may be an underestimate. */ +#define UPB_PB_ENCODER_SIZE 784 + +struct upb_pb_encoder; +typedef struct upb_pb_encoder upb_pb_encoder; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_sink upb_pb_encoder_input(upb_pb_encoder *p); +upb_pb_encoder* upb_pb_encoder_create(upb_arena* a, const upb_handlers* h, + upb_bytessink output); + +/* Lazily builds and caches handlers that will push encoded data to a bytessink. + * Any msgdef objects used with this object must outlive it. */ +upb_handlercache *upb_pb_encoder_newcache(void); + +#ifdef __cplusplus +} /* extern "C" { */ + +class upb::pb::EncoderPtr { + public: + EncoderPtr(upb_pb_encoder* ptr) : ptr_(ptr) {} + + upb_pb_encoder* ptr() { return ptr_; } + + /* Creates a new encoder in the given environment. The Handlers must have + * come from NewHandlers() below. */ + static EncoderPtr Create(Arena* arena, const Handlers* handlers, + BytesSink output) { + return EncoderPtr( + upb_pb_encoder_create(arena->ptr(), handlers, output.sink())); + } + + /* The input to the encoder. */ + upb::Sink input() { return upb_pb_encoder_input(ptr()); } + + /* Creates a new set of handlers for this MessageDef. */ + static HandlerCache NewCache() { + return HandlerCache(upb_pb_encoder_newcache()); + } + + static const size_t kSize = UPB_PB_ENCODER_SIZE; + + private: + upb_pb_encoder* ptr_; +}; + +#endif /* __cplusplus */ + +#endif /* UPB_ENCODER_H_ */ diff --git a/third_party/upb/upb/pb/make-gdb-script.rb b/third_party/upb/upb/pb/make-gdb-script.rb new file mode 100755 index 00000000000..3895597887d --- /dev/null +++ b/third_party/upb/upb/pb/make-gdb-script.rb @@ -0,0 +1,36 @@ +#!/usr/bin/ruby + +puts "set width 0 +set height 0 +set verbose off\n\n" + +IO.popen("nm -S /tmp/upb-jit-code.so").each_line { |line| + # Input lines look like this: + # 000000000000575a T X.0x10.OP_CHECKDELIM + # + # For each one we want to emit a command that looks like: + # b X.0x10.OP_CHECKDELIM + # commands + # silent + # printf "buf_ofs=%d data_rem=%d delim_rem=%d X.0x10.OP_CHECKDELIM\n", $rbx - (long)((upb_pbdecoder*)($r15))->buf, $r12 - $rbx, $rbp - $rbx + # continue + # end + + parts = line.split + next if parts[1] != "T" + sym = parts[2] + next if sym !~ /X\./; + if sym =~ /OP_/ then + printcmd = "printf \"buf_ofs=%d data_rem=%d delim_rem=%d #{sym}\\n\", $rbx - (long)((upb_pbdecoder*)($r15))->buf, $r12 - $rbx, $rbp - $rbx" + elsif sym =~ /enterjit/ then + printcmd = "printf \"#{sym} bytes=%d\\n\", $rcx" + else + printcmd = "printf \"#{sym}\\n\"" + end + puts "b #{sym} +commands + silent + #{printcmd} + continue +end\n\n" +} diff --git a/third_party/upb/upb/pb/textprinter.c b/third_party/upb/upb/pb/textprinter.c new file mode 100644 index 00000000000..0760173e43c --- /dev/null +++ b/third_party/upb/upb/pb/textprinter.c @@ -0,0 +1,340 @@ +/* + * upb::pb::TextPrinter + * + * OPT: This is not optimized at all. It uses printf() which parses the format + * string every time, and it allocates memory for every put. + */ + +#include "upb/pb/textprinter.h" + +#include +#include +#include +#include +#include +#include + +#include "upb/sink.h" + +#include "upb/port_def.inc" + +struct upb_textprinter { + upb_sink input_; + upb_bytessink output_; + int indent_depth_; + bool single_line_; + void *subc; +}; + +#define CHECK(x) if ((x) < 0) goto err; + +static const char *shortname(const char *longname) { + const char *last = strrchr(longname, '.'); + return last ? last + 1 : longname; +} + +static int indent(upb_textprinter *p) { + int i; + if (!p->single_line_) + for (i = 0; i < p->indent_depth_; i++) + upb_bytessink_putbuf(p->output_, p->subc, " ", 2, NULL); + return 0; +} + +static int endfield(upb_textprinter *p) { + const char ch = (p->single_line_ ? ' ' : '\n'); + upb_bytessink_putbuf(p->output_, p->subc, &ch, 1, NULL); + return 0; +} + +static int putescaped(upb_textprinter *p, const char *buf, size_t len, + bool preserve_utf8) { + /* Based on CEscapeInternal() from Google's protobuf release. */ + char dstbuf[4096], *dst = dstbuf, *dstend = dstbuf + sizeof(dstbuf); + const char *end = buf + len; + + /* I think hex is prettier and more useful, but proto2 uses octal; should + * investigate whether it can parse hex also. */ + const bool use_hex = false; + bool last_hex_escape = false; /* true if last output char was \xNN */ + + for (; buf < end; buf++) { + bool is_hex_escape; + + if (dstend - dst < 4) { + upb_bytessink_putbuf(p->output_, p->subc, dstbuf, dst - dstbuf, NULL); + dst = dstbuf; + } + + is_hex_escape = false; + switch (*buf) { + case '\n': *(dst++) = '\\'; *(dst++) = 'n'; break; + case '\r': *(dst++) = '\\'; *(dst++) = 'r'; break; + case '\t': *(dst++) = '\\'; *(dst++) = 't'; break; + case '\"': *(dst++) = '\\'; *(dst++) = '\"'; break; + case '\'': *(dst++) = '\\'; *(dst++) = '\''; break; + case '\\': *(dst++) = '\\'; *(dst++) = '\\'; break; + default: + /* Note that if we emit \xNN and the buf character after that is a hex + * digit then that digit must be escaped too to prevent it being + * interpreted as part of the character code by C. */ + if ((!preserve_utf8 || (uint8_t)*buf < 0x80) && + (!isprint(*buf) || (last_hex_escape && isxdigit(*buf)))) { + sprintf(dst, (use_hex ? "\\x%02x" : "\\%03o"), (uint8_t)*buf); + is_hex_escape = use_hex; + dst += 4; + } else { + *(dst++) = *buf; break; + } + } + last_hex_escape = is_hex_escape; + } + /* Flush remaining data. */ + upb_bytessink_putbuf(p->output_, p->subc, dstbuf, dst - dstbuf, NULL); + return 0; +} + +bool putf(upb_textprinter *p, const char *fmt, ...) { + va_list args; + va_list args_copy; + char *str; + int written; + int len; + bool ok; + + va_start(args, fmt); + + /* Run once to get the length of the string. */ + _upb_va_copy(args_copy, args); + len = _upb_vsnprintf(NULL, 0, fmt, args_copy); + va_end(args_copy); + + /* + 1 for NULL terminator (vsprintf() requires it even if we don't). */ + str = upb_gmalloc(len + 1); + if (!str) return false; + written = vsprintf(str, fmt, args); + va_end(args); + UPB_ASSERT(written == len); + + ok = upb_bytessink_putbuf(p->output_, p->subc, str, len, NULL); + upb_gfree(str); + return ok; +} + + +/* handlers *******************************************************************/ + +static bool textprinter_startmsg(void *c, const void *hd) { + upb_textprinter *p = c; + UPB_UNUSED(hd); + if (p->indent_depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc); + } + return true; +} + +static bool textprinter_endmsg(void *c, const void *hd, upb_status *s) { + upb_textprinter *p = c; + UPB_UNUSED(hd); + UPB_UNUSED(s); + if (p->indent_depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +#define TYPE(name, ctype, fmt) \ + static bool textprinter_put ## name(void *closure, const void *handler_data, \ + ctype val) { \ + upb_textprinter *p = closure; \ + const upb_fielddef *f = handler_data; \ + CHECK(indent(p)); \ + putf(p, "%s: " fmt, upb_fielddef_name(f), val); \ + CHECK(endfield(p)); \ + return true; \ + err: \ + return false; \ +} + +static bool textprinter_putbool(void *closure, const void *handler_data, + bool val) { + upb_textprinter *p = closure; + const upb_fielddef *f = handler_data; + CHECK(indent(p)); + putf(p, "%s: %s", upb_fielddef_name(f), val ? "true" : "false"); + CHECK(endfield(p)); + return true; +err: + return false; +} + +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY_MACROVAL(x) STRINGIFY_HELPER(x) + +TYPE(int32, int32_t, "%" PRId32) +TYPE(int64, int64_t, "%" PRId64) +TYPE(uint32, uint32_t, "%" PRIu32) +TYPE(uint64, uint64_t, "%" PRIu64) +TYPE(float, float, "%." STRINGIFY_MACROVAL(FLT_DIG) "g") +TYPE(double, double, "%." STRINGIFY_MACROVAL(DBL_DIG) "g") + +#undef TYPE + +/* Output a symbolic value from the enum if found, else just print as int32. */ +static bool textprinter_putenum(void *closure, const void *handler_data, + int32_t val) { + upb_textprinter *p = closure; + const upb_fielddef *f = handler_data; + const upb_enumdef *enum_def = upb_fielddef_enumsubdef(f); + const char *label = upb_enumdef_iton(enum_def, val); + if (label) { + indent(p); + putf(p, "%s: %s", upb_fielddef_name(f), label); + endfield(p); + } else { + if (!textprinter_putint32(closure, handler_data, val)) + return false; + } + return true; +} + +static void *textprinter_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_textprinter *p = closure; + const upb_fielddef *f = handler_data; + UPB_UNUSED(size_hint); + indent(p); + putf(p, "%s: \"", upb_fielddef_name(f)); + return p; +} + +static bool textprinter_endstr(void *closure, const void *handler_data) { + upb_textprinter *p = closure; + UPB_UNUSED(handler_data); + putf(p, "\""); + endfield(p); + return true; +} + +static size_t textprinter_putstr(void *closure, const void *hd, const char *buf, + size_t len, const upb_bufhandle *handle) { + upb_textprinter *p = closure; + const upb_fielddef *f = hd; + UPB_UNUSED(handle); + CHECK(putescaped(p, buf, len, upb_fielddef_type(f) == UPB_TYPE_STRING)); + return len; +err: + return 0; +} + +static void *textprinter_startsubmsg(void *closure, const void *handler_data) { + upb_textprinter *p = closure; + const char *name = handler_data; + CHECK(indent(p)); + putf(p, "%s {%c", name, p->single_line_ ? ' ' : '\n'); + p->indent_depth_++; + return p; +err: + return UPB_BREAK; +} + +static bool textprinter_endsubmsg(void *closure, const void *handler_data) { + upb_textprinter *p = closure; + UPB_UNUSED(handler_data); + p->indent_depth_--; + CHECK(indent(p)); + upb_bytessink_putbuf(p->output_, p->subc, "}", 1, NULL); + CHECK(endfield(p)); + return true; +err: + return false; +} + +static void onmreg(const void *c, upb_handlers *h) { + const upb_msgdef *m = upb_handlers_msgdef(h); + upb_msg_field_iter i; + UPB_UNUSED(c); + + upb_handlers_setstartmsg(h, textprinter_startmsg, NULL); + upb_handlers_setendmsg(h, textprinter_endmsg, NULL); + + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + attr.handler_data = f; + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, f, textprinter_putint32, &attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, f, textprinter_putint64, &attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, f, textprinter_putuint32, &attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, f, textprinter_putuint64, &attr); + break; + case UPB_TYPE_FLOAT: + upb_handlers_setfloat(h, f, textprinter_putfloat, &attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, f, textprinter_putdouble, &attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, f, textprinter_putbool, &attr); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + upb_handlers_setstartstr(h, f, textprinter_startstr, &attr); + upb_handlers_setstring(h, f, textprinter_putstr, &attr); + upb_handlers_setendstr(h, f, textprinter_endstr, &attr); + break; + case UPB_TYPE_MESSAGE: { + const char *name = + upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_GROUP + ? shortname(upb_msgdef_fullname(upb_fielddef_msgsubdef(f))) + : upb_fielddef_name(f); + attr.handler_data = name; + upb_handlers_setstartsubmsg(h, f, textprinter_startsubmsg, &attr); + upb_handlers_setendsubmsg(h, f, textprinter_endsubmsg, &attr); + break; + } + case UPB_TYPE_ENUM: + upb_handlers_setint32(h, f, textprinter_putenum, &attr); + break; + } + } +} + +static void textprinter_reset(upb_textprinter *p, bool single_line) { + p->single_line_ = single_line; + p->indent_depth_ = 0; +} + + +/* Public API *****************************************************************/ + +upb_textprinter *upb_textprinter_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output) { + upb_textprinter *p = upb_arena_malloc(arena, sizeof(upb_textprinter)); + if (!p) return NULL; + + p->output_ = output; + upb_sink_reset(&p->input_, h, p); + textprinter_reset(p, false); + + return p; +} + +upb_handlercache *upb_textprinter_newcache(void) { + return upb_handlercache_new(&onmreg, NULL); +} + +upb_sink upb_textprinter_input(upb_textprinter *p) { return p->input_; } + +void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line) { + p->single_line_ = single_line; +} diff --git a/third_party/upb/upb/pb/textprinter.h b/third_party/upb/upb/pb/textprinter.h new file mode 100644 index 00000000000..7e20d7521bf --- /dev/null +++ b/third_party/upb/upb/pb/textprinter.h @@ -0,0 +1,69 @@ +/* +** upb::pb::TextPrinter (upb_textprinter) +** +** Handlers for writing to protobuf text format. +*/ + +#ifndef UPB_TEXT_H_ +#define UPB_TEXT_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace pb { +class TextPrinterPtr; +} /* namespace pb */ +} /* namespace upb */ +#endif + +/* upb_textprinter ************************************************************/ + +struct upb_textprinter; +typedef struct upb_textprinter upb_textprinter; + +#ifdef __cplusplus +extern "C" { +#endif + +/* C API. */ +upb_textprinter *upb_textprinter_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output); +void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line); +upb_sink upb_textprinter_input(upb_textprinter *p); +upb_handlercache *upb_textprinter_newcache(void); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::pb::TextPrinterPtr { + public: + TextPrinterPtr(upb_textprinter* ptr) : ptr_(ptr) {} + + /* The given handlers must have come from NewHandlers(). It must outlive the + * TextPrinter. */ + static TextPrinterPtr Create(Arena *arena, upb::HandlersPtr *handlers, + BytesSink output) { + return TextPrinterPtr( + upb_textprinter_create(arena->ptr(), handlers->ptr(), output.sink())); + } + + void SetSingleLineMode(bool single_line) { + upb_textprinter_setsingleline(ptr_, single_line); + } + + Sink input() { return upb_textprinter_input(ptr_); } + + /* If handler caching becomes a requirement we can add a code cache as in + * decoder.h */ + static HandlerCache NewCache() { + return HandlerCache(upb_textprinter_newcache()); + } + + private: + upb_textprinter* ptr_; +}; + +#endif + +#endif /* UPB_TEXT_H_ */ diff --git a/third_party/upb/upb/pb/varint.c b/third_party/upb/upb/pb/varint.c new file mode 100644 index 00000000000..90f58a138fc --- /dev/null +++ b/third_party/upb/upb/pb/varint.c @@ -0,0 +1,74 @@ + +#include "upb/pb/varint.int.h" + +/* Index is descriptor type. */ +const uint8_t upb_pb_native_wire_types[] = { + UPB_WIRE_TYPE_END_GROUP, /* ENDGROUP */ + UPB_WIRE_TYPE_64BIT, /* DOUBLE */ + UPB_WIRE_TYPE_32BIT, /* FLOAT */ + UPB_WIRE_TYPE_VARINT, /* INT64 */ + UPB_WIRE_TYPE_VARINT, /* UINT64 */ + UPB_WIRE_TYPE_VARINT, /* INT32 */ + UPB_WIRE_TYPE_64BIT, /* FIXED64 */ + UPB_WIRE_TYPE_32BIT, /* FIXED32 */ + UPB_WIRE_TYPE_VARINT, /* BOOL */ + UPB_WIRE_TYPE_DELIMITED, /* STRING */ + UPB_WIRE_TYPE_START_GROUP, /* GROUP */ + UPB_WIRE_TYPE_DELIMITED, /* MESSAGE */ + UPB_WIRE_TYPE_DELIMITED, /* BYTES */ + UPB_WIRE_TYPE_VARINT, /* UINT32 */ + UPB_WIRE_TYPE_VARINT, /* ENUM */ + UPB_WIRE_TYPE_32BIT, /* SFIXED32 */ + UPB_WIRE_TYPE_64BIT, /* SFIXED64 */ + UPB_WIRE_TYPE_VARINT, /* SINT32 */ + UPB_WIRE_TYPE_VARINT, /* SINT64 */ +}; + +/* A basic branch-based decoder, uses 32-bit values to get good performance + * on 32-bit architectures (but performs well on 64-bits also). + * This scheme comes from the original Google Protobuf implementation + * (proto2). */ +upb_decoderet upb_vdecode_max8_branch32(upb_decoderet r) { + upb_decoderet err = {NULL, 0}; + const char *p = r.p; + uint32_t low = (uint32_t)r.val; + uint32_t high = 0; + uint32_t b; + b = *(p++); low |= (b & 0x7fU) << 14; if (!(b & 0x80)) goto done; + b = *(p++); low |= (b & 0x7fU) << 21; if (!(b & 0x80)) goto done; + b = *(p++); low |= (b & 0x7fU) << 28; + high = (b & 0x7fU) >> 4; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 3; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 10; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 17; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 24; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 31; if (!(b & 0x80)) goto done; + return err; + +done: + r.val = ((uint64_t)high << 32) | low; + r.p = p; + return r; +} + +/* Like the previous, but uses 64-bit values. */ +upb_decoderet upb_vdecode_max8_branch64(upb_decoderet r) { + const char *p = r.p; + uint64_t val = r.val; + uint64_t b; + upb_decoderet err = {NULL, 0}; + b = *(p++); val |= (b & 0x7fU) << 14; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 21; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 28; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 35; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 42; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 49; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 56; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 63; if (!(b & 0x80)) goto done; + return err; + +done: + r.val = val; + r.p = p; + return r; +} diff --git a/third_party/upb/upb/pb/varint.int.h b/third_party/upb/upb/pb/varint.int.h new file mode 100644 index 00000000000..ff1ca661491 --- /dev/null +++ b/third_party/upb/upb/pb/varint.int.h @@ -0,0 +1,164 @@ +/* +** A number of routines for varint manipulation (we keep them all around to +** have multiple approaches available for benchmarking). +*/ + +#ifndef UPB_VARINT_DECODER_H_ +#define UPB_VARINT_DECODER_H_ + +#include +#include +#include +#include "upb/upb.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +#define UPB_MAX_WIRE_TYPE 5 + +/* The maximum number of bytes that it takes to encode a 64-bit varint. */ +#define UPB_PB_VARINT_MAX_LEN 10 + +/* Array of the "native" (ie. non-packed-repeated) wire type for the given a + * descriptor type (upb_descriptortype_t). */ +extern const uint8_t upb_pb_native_wire_types[]; + +UPB_INLINE uint64_t byteswap64(uint64_t val) +{ + return ((((val) & 0xff00000000000000ull) >> 56) + | (((val) & 0x00ff000000000000ull) >> 40) + | (((val) & 0x0000ff0000000000ull) >> 24) + | (((val) & 0x000000ff00000000ull) >> 8) + | (((val) & 0x00000000ff000000ull) << 8) + | (((val) & 0x0000000000ff0000ull) << 24) + | (((val) & 0x000000000000ff00ull) << 40) + | (((val) & 0x00000000000000ffull) << 56)); +} + +/* Zig-zag encoding/decoding **************************************************/ + +UPB_INLINE int32_t upb_zzdec_32(uint32_t n) { + return (n >> 1) ^ -(int32_t)(n & 1); +} +UPB_INLINE int64_t upb_zzdec_64(uint64_t n) { + return (n >> 1) ^ -(int64_t)(n & 1); +} +UPB_INLINE uint32_t upb_zzenc_32(int32_t n) { + return ((uint32_t)n << 1) ^ (n >> 31); +} +UPB_INLINE uint64_t upb_zzenc_64(int64_t n) { + return ((uint64_t)n << 1) ^ (n >> 63); +} + +/* Decoding *******************************************************************/ + +/* All decoding functions return this struct by value. */ +typedef struct { + const char *p; /* NULL if the varint was unterminated. */ + uint64_t val; +} upb_decoderet; + +UPB_INLINE upb_decoderet upb_decoderet_make(const char *p, uint64_t val) { + upb_decoderet ret; + ret.p = p; + ret.val = val; + return ret; +} + +upb_decoderet upb_vdecode_max8_branch32(upb_decoderet r); +upb_decoderet upb_vdecode_max8_branch64(upb_decoderet r); + +/* Template for a function that checks the first two bytes with branching + * and dispatches 2-10 bytes with a separate function. Note that this may read + * up to 10 bytes, so it must not be used unless there are at least ten bytes + * left in the buffer! */ +#define UPB_VARINT_DECODER_CHECK2(name, decode_max8_function) \ +UPB_INLINE upb_decoderet upb_vdecode_check2_ ## name(const char *_p) { \ + uint8_t *p = (uint8_t*)_p; \ + upb_decoderet r; \ + if ((*p & 0x80) == 0) { \ + /* Common case: one-byte varint. */ \ + return upb_decoderet_make(_p + 1, *p & 0x7fU); \ + } \ + r = upb_decoderet_make(_p + 2, (*p & 0x7fU) | ((*(p + 1) & 0x7fU) << 7)); \ + if ((*(p + 1) & 0x80) == 0) { \ + /* Two-byte varint. */ \ + return r; \ + } \ + /* Longer varint, fallback to out-of-line function. */ \ + return decode_max8_function(r); \ +} + +UPB_VARINT_DECODER_CHECK2(branch32, upb_vdecode_max8_branch32) +UPB_VARINT_DECODER_CHECK2(branch64, upb_vdecode_max8_branch64) +#undef UPB_VARINT_DECODER_CHECK2 + +/* Our canonical functions for decoding varints, based on the currently + * favored best-performing implementations. */ +UPB_INLINE upb_decoderet upb_vdecode_fast(const char *p) { + if (sizeof(long) == 8) + return upb_vdecode_check2_branch64(p); + else + return upb_vdecode_check2_branch32(p); +} + + +/* Encoding *******************************************************************/ + +UPB_INLINE int upb_value_size(uint64_t val) { +#ifdef __GNUC__ + int high_bit = 63 - __builtin_clzll(val); /* 0-based, undef if val == 0. */ +#else + int high_bit = 0; + uint64_t tmp = val; + while(tmp >>= 1) high_bit++; +#endif + return val == 0 ? 1 : high_bit / 8 + 1; +} + +/* Encodes a 64-bit varint into buf (which must be >=UPB_PB_VARINT_MAX_LEN + * bytes long), returning how many bytes were used. + * + * TODO: benchmark and optimize if necessary. */ +UPB_INLINE size_t upb_vencode64(uint64_t val, char *buf) { + size_t i; + if (val == 0) { buf[0] = 0; return 1; } + i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +UPB_INLINE size_t upb_varint_size(uint64_t val) { + char buf[UPB_PB_VARINT_MAX_LEN]; + return upb_vencode64(val, buf); +} + +/* Encodes a 32-bit varint, *not* sign-extended. */ +UPB_INLINE uint64_t upb_vencode32(uint32_t val) { + char buf[UPB_PB_VARINT_MAX_LEN]; + size_t bytes = upb_vencode64(val, buf); + uint64_t ret = 0; + UPB_ASSERT(bytes <= 5); + memcpy(&ret, buf, bytes); +#ifdef UPB_BIG_ENDIAN + ret = byteswap64(ret); +#endif + UPB_ASSERT(ret <= 0xffffffffffU); + return ret; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* UPB_VARINT_DECODER_H_ */ diff --git a/third_party/upb/upb/port.c b/third_party/upb/upb/port.c new file mode 100644 index 00000000000..023f7dc0ce1 --- /dev/null +++ b/third_party/upb/upb/port.c @@ -0,0 +1,27 @@ + +#include "upb/upb.h" +#include "upb/port_def.inc" + +#ifdef UPB_MSVC_VSNPRINTF +/* Visual C++ earlier than 2015 doesn't have standard C99 snprintf and + * vsnprintf. To support them, missing functions are manually implemented + * using the existing secure functions. */ +int msvc_vsnprintf(char* s, size_t n, const char* format, va_list arg) { + if (!s) { + return _vscprintf(format, arg); + } + int ret = _vsnprintf_s(s, n, _TRUNCATE, format, arg); + if (ret < 0) { + ret = _vscprintf(format, arg); + } + return ret; +} + +int msvc_snprintf(char* s, size_t n, const char* format, ...) { + va_list arg; + va_start(arg, format); + int ret = msvc_vsnprintf(s, n, format, arg); + va_end(arg); + return ret; +} +#endif diff --git a/third_party/upb/upb/port_def.inc b/third_party/upb/upb/port_def.inc new file mode 100644 index 00000000000..a8967b36794 --- /dev/null +++ b/third_party/upb/upb/port_def.inc @@ -0,0 +1,152 @@ +/* +* This is where we define macros used across upb. +* +* All of these macros are undef'd in port_undef.inc to avoid leaking them to +* users. +* +* The correct usage is: +* +* #include "upb/foobar.h" +* #include "upb/baz.h" +* +* // MUST be last included header. +* #include "upb/port_def.inc" +* +* // Code for this file. +* // <...> +* +* // Can be omitted for .c files, required for .h. +* #include "upb/port_undef.inc" +* +* This file is private and must not be included by users! +*/ +#ifndef UINTPTR_MAX +#error must include stdint.h first +#endif + +#if UINTPTR_MAX == 0xffffffff +#define UPB_SIZE(size32, size64) size32 +#else +#define UPB_SIZE(size32, size64) size64 +#endif + +#define UPB_FIELD_AT(msg, fieldtype, offset) \ + *(fieldtype*)((const char*)(msg) + offset) + +#define UPB_READ_ONEOF(msg, fieldtype, offset, case_offset, case_val, default) \ + UPB_FIELD_AT(msg, int, case_offset) == case_val \ + ? UPB_FIELD_AT(msg, fieldtype, offset) \ + : default + +#define UPB_WRITE_ONEOF(msg, fieldtype, offset, value, case_offset, case_val) \ + UPB_FIELD_AT(msg, int, case_offset) = case_val; \ + UPB_FIELD_AT(msg, fieldtype, offset) = value; + +/* UPB_INLINE: inline if possible, emit standalone code if required. */ +#ifdef __cplusplus +#define UPB_INLINE inline +#elif defined (__GNUC__) || defined(__clang__) +#define UPB_INLINE static __inline__ +#else +#define UPB_INLINE static +#endif + +/* Hints to the compiler about likely/unlikely branches. */ +#if defined (__GNUC__) || defined(__clang__) +#define UPB_LIKELY(x) __builtin_expect((x),1) +#define UPB_UNLIKELY(x) __builtin_expect((x),0) +#else +#define UPB_LIKELY(x) (x) +#define UPB_UNLIKELY(x) (x) +#endif + +/* Define UPB_BIG_ENDIAN manually if you're on big endian and your compiler + * doesn't provide these preprocessor symbols. */ +#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define UPB_BIG_ENDIAN +#endif + +/* Macros for function attributes on compilers that support them. */ +#ifdef __GNUC__ +#define UPB_FORCEINLINE __inline__ __attribute__((always_inline)) +#define UPB_NOINLINE __attribute__((noinline)) +#define UPB_NORETURN __attribute__((__noreturn__)) +#else /* !defined(__GNUC__) */ +#define UPB_FORCEINLINE +#define UPB_NOINLINE +#define UPB_NORETURN +#endif + +#if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L +/* C99/C++11 versions. */ +#include +#define _upb_snprintf snprintf +#define _upb_vsnprintf vsnprintf +#define _upb_va_copy(a, b) va_copy(a, b) +#elif defined(_MSC_VER) +/* Microsoft C/C++ versions. */ +#include +#include +#if _MSC_VER < 1900 +int msvc_snprintf(char* s, size_t n, const char* format, ...); +int msvc_vsnprintf(char* s, size_t n, const char* format, va_list arg); +#define UPB_MSVC_VSNPRINTF +#define _upb_snprintf msvc_snprintf +#define _upb_vsnprintf msvc_vsnprintf +#else +#define _upb_snprintf snprintf +#define _upb_vsnprintf vsnprintf +#endif +#define _upb_va_copy(a, b) va_copy(a, b) +#elif defined __GNUC__ +/* A few hacky workarounds for functions not in C89. + * For internal use only! + * TODO(haberman): fix these by including our own implementations, or finding + * another workaround. + */ +#define _upb_snprintf __builtin_snprintf +#define _upb_vsnprintf __builtin_vsnprintf +#define _upb_va_copy(a, b) __va_copy(a, b) +#else +#error Need implementations of [v]snprintf and va_copy +#endif + +#ifdef __cplusplus +#if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__) || \ + (defined(_MSC_VER) && _MSC_VER >= 1900) +/* C++11 is present */ +#else +#error upb requires C++11 for C++ support +#endif +#endif + +#define UPB_MAX(x, y) ((x) > (y) ? (x) : (y)) +#define UPB_MIN(x, y) ((x) < (y) ? (x) : (y)) + +#define UPB_UNUSED(var) (void)var + +/* UPB_ASSERT(): in release mode, we use the expression without letting it be + * evaluated. This prevents "unused variable" warnings. */ +#ifdef NDEBUG +#define UPB_ASSERT(expr) do {} while (false && (expr)) +#else +#define UPB_ASSERT(expr) assert(expr) +#endif + +/* UPB_ASSERT_DEBUGVAR(): assert that uses functions or variables that only + * exist in debug mode. This turns into regular assert. */ +#define UPB_ASSERT_DEBUGVAR(expr) assert(expr) + +#if defined(__GNUC__) || defined(__clang__) +#define UPB_UNREACHABLE() do { assert(0); __builtin_unreachable(); } while(0) +#else +#define UPB_UNREACHABLE() do { assert(0); } while(0) +#endif + +/* UPB_INFINITY representing floating-point positive infinity. */ +#include +#ifdef INFINITY +#define UPB_INFINITY INFINITY +#else +#define UPB_INFINITY (1.0 / 0.0) +#endif diff --git a/third_party/upb/upb/port_undef.inc b/third_party/upb/upb/port_undef.inc new file mode 100644 index 00000000000..103180b7fbd --- /dev/null +++ b/third_party/upb/upb/port_undef.inc @@ -0,0 +1,21 @@ +/* See port_def.inc. This should #undef all macros #defined there. */ + +#undef UPB_SIZE +#undef UPB_FIELD_AT +#undef UPB_READ_ONEOF +#undef UPB_WRITE_ONEOF +#undef UPB_INLINE +#undef UPB_FORCEINLINE +#undef UPB_NOINLINE +#undef UPB_NORETURN +#undef UPB_MAX +#undef UPB_MIN +#undef UPB_UNUSED +#undef UPB_ASSERT +#undef UPB_ASSERT_DEBUGVAR +#undef UPB_UNREACHABLE +#undef UPB_INFINITY +#undef UPB_MSVC_VSNPRINTF +#undef _upb_snprintf +#undef _upb_vsnprintf +#undef _upb_va_copy diff --git a/third_party/upb/upb/sink.c b/third_party/upb/upb/sink.c new file mode 100644 index 00000000000..d55d258b235 --- /dev/null +++ b/third_party/upb/upb/sink.c @@ -0,0 +1,17 @@ + +#include "upb/sink.h" + +bool upb_bufsrc_putbuf(const char *buf, size_t len, upb_bytessink sink) { + void *subc; + bool ret; + upb_bufhandle handle = UPB_BUFHANDLE_INIT; + handle.buf = buf; + ret = upb_bytessink_start(sink, len, &subc); + if (ret && len != 0) { + ret = (upb_bytessink_putbuf(sink, subc, buf, len, &handle) >= len); + } + if (ret) { + ret = upb_bytessink_end(sink); + } + return ret; +} diff --git a/third_party/upb/upb/sink.h b/third_party/upb/upb/sink.h new file mode 100644 index 00000000000..47d218a9c1b --- /dev/null +++ b/third_party/upb/upb/sink.h @@ -0,0 +1,516 @@ +/* +** upb::Sink (upb_sink) +** upb::BytesSink (upb_bytessink) +** +** A upb_sink is an object that binds a upb_handlers object to some runtime +** state. It is the object that can actually receive data via the upb_handlers +** interface. +** +** Unlike upb_def and upb_handlers, upb_sink is never frozen, immutable, or +** thread-safe. You can create as many of them as you want, but each one may +** only be used in a single thread at a time. +** +** If we compare with class-based OOP, a you can think of a upb_def as an +** abstract base class, a upb_handlers as a concrete derived class, and a +** upb_sink as an object (class instance). +*/ + +#ifndef UPB_SINK_H +#define UPB_SINK_H + +#include "upb/handlers.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +namespace upb { +class BytesSink; +class Sink; +} +#endif + +/* upb_sink *******************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + const upb_handlers *handlers; + void *closure; +} upb_sink; + +#define PUTVAL(type, ctype) \ + UPB_INLINE bool upb_sink_put##type(upb_sink s, upb_selector_t sel, \ + ctype val) { \ + typedef upb_##type##_handlerfunc functype; \ + functype *func; \ + const void *hd; \ + if (!s.handlers) return true; \ + func = (functype *)upb_handlers_gethandler(s.handlers, sel, &hd); \ + if (!func) return true; \ + return func(s.closure, hd, val); \ + } + +PUTVAL(int32, int32_t) +PUTVAL(int64, int64_t) +PUTVAL(uint32, uint32_t) +PUTVAL(uint64, uint64_t) +PUTVAL(float, float) +PUTVAL(double, double) +PUTVAL(bool, bool) +#undef PUTVAL + +UPB_INLINE void upb_sink_reset(upb_sink *s, const upb_handlers *h, void *c) { + s->handlers = h; + s->closure = c; +} + +UPB_INLINE size_t upb_sink_putstring(upb_sink s, upb_selector_t sel, + const char *buf, size_t n, + const upb_bufhandle *handle) { + typedef upb_string_handlerfunc func; + func *handler; + const void *hd; + if (!s.handlers) return n; + handler = (func *)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!handler) return n; + return handler(s.closure, hd, buf, n, handle); +} + +UPB_INLINE bool upb_sink_putunknown(upb_sink s, const char *buf, size_t n) { + typedef upb_unknown_handlerfunc func; + func *handler; + const void *hd; + if (!s.handlers) return true; + handler = + (func *)upb_handlers_gethandler(s.handlers, UPB_UNKNOWN_SELECTOR, &hd); + + if (!handler) return n; + return handler(s.closure, hd, buf, n); +} + +UPB_INLINE bool upb_sink_startmsg(upb_sink s) { + typedef upb_startmsg_handlerfunc func; + func *startmsg; + const void *hd; + if (!s.handlers) return true; + startmsg = + (func *)upb_handlers_gethandler(s.handlers, UPB_STARTMSG_SELECTOR, &hd); + + if (!startmsg) return true; + return startmsg(s.closure, hd); +} + +UPB_INLINE bool upb_sink_endmsg(upb_sink s, upb_status *status) { + typedef upb_endmsg_handlerfunc func; + func *endmsg; + const void *hd; + if (!s.handlers) return true; + endmsg = + (func *)upb_handlers_gethandler(s.handlers, UPB_ENDMSG_SELECTOR, &hd); + + if (!endmsg) return true; + return endmsg(s.closure, hd, status); +} + +UPB_INLINE bool upb_sink_startseq(upb_sink s, upb_selector_t sel, + upb_sink *sub) { + typedef upb_startfield_handlerfunc func; + func *startseq; + const void *hd; + sub->closure = s.closure; + sub->handlers = s.handlers; + if (!s.handlers) return true; + startseq = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!startseq) return true; + sub->closure = startseq(s.closure, hd); + return sub->closure ? true : false; +} + +UPB_INLINE bool upb_sink_endseq(upb_sink s, upb_selector_t sel) { + typedef upb_endfield_handlerfunc func; + func *endseq; + const void *hd; + if (!s.handlers) return true; + endseq = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!endseq) return true; + return endseq(s.closure, hd); +} + +UPB_INLINE bool upb_sink_startstr(upb_sink s, upb_selector_t sel, + size_t size_hint, upb_sink *sub) { + typedef upb_startstr_handlerfunc func; + func *startstr; + const void *hd; + sub->closure = s.closure; + sub->handlers = s.handlers; + if (!s.handlers) return true; + startstr = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!startstr) return true; + sub->closure = startstr(s.closure, hd, size_hint); + return sub->closure ? true : false; +} + +UPB_INLINE bool upb_sink_endstr(upb_sink s, upb_selector_t sel) { + typedef upb_endfield_handlerfunc func; + func *endstr; + const void *hd; + if (!s.handlers) return true; + endstr = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!endstr) return true; + return endstr(s.closure, hd); +} + +UPB_INLINE bool upb_sink_startsubmsg(upb_sink s, upb_selector_t sel, + upb_sink *sub) { + typedef upb_startfield_handlerfunc func; + func *startsubmsg; + const void *hd; + sub->closure = s.closure; + if (!s.handlers) { + sub->handlers = NULL; + return true; + } + sub->handlers = upb_handlers_getsubhandlers_sel(s.handlers, sel); + startsubmsg = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!startsubmsg) return true; + sub->closure = startsubmsg(s.closure, hd); + return sub->closure ? true : false; +} + +UPB_INLINE bool upb_sink_endsubmsg(upb_sink s, upb_selector_t sel) { + typedef upb_endfield_handlerfunc func; + func *endsubmsg; + const void *hd; + if (!s.handlers) return true; + endsubmsg = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!endsubmsg) return s.closure; + return endsubmsg(s.closure, hd); +} + +#ifdef __cplusplus +} /* extern "C" */ + +/* A upb::Sink is an object that binds a upb::Handlers object to some runtime + * state. It represents an endpoint to which data can be sent. + * + * TODO(haberman): right now all of these functions take selectors. Should they + * take selectorbase instead? + * + * ie. instead of calling: + * sink->StartString(FOO_FIELD_START_STRING, ...) + * a selector base would let you say: + * sink->StartString(FOO_FIELD, ...) + * + * This would make call sites a little nicer and require emitting fewer selector + * definitions in .h files. + * + * But the current scheme has the benefit that you can retrieve a function + * pointer for any handler with handlers->GetHandler(selector), without having + * to have a separate GetHandler() function for each handler type. The JIT + * compiler uses this. To accommodate we'd have to expose a separate + * GetHandler() for every handler type. + * + * Also to ponder: selectors right now are independent of a specific Handlers + * instance. In other words, they allocate a number to every possible handler + * that *could* be registered, without knowing anything about what handlers + * *are* registered. That means that using selectors as table offsets prohibits + * us from compacting the handler table at Freeze() time. If the table is very + * sparse, this could be wasteful. + * + * Having another selector-like thing that is specific to a Handlers instance + * would allow this compacting, but then it would be impossible to write code + * ahead-of-time that can be bound to any Handlers instance at runtime. For + * example, a .proto file parser written as straight C will not know what + * Handlers it will be bound to, so when it calls sink->StartString() what + * selector will it pass? It needs a selector like we have today, that is + * independent of any particular upb::Handlers. + * + * Is there a way then to allow Handlers table compaction? */ +class upb::Sink { + public: + /* Constructor with no initialization; must be Reset() before use. */ + Sink() {} + + Sink(const Sink&) = default; + Sink& operator=(const Sink&) = default; + + Sink(const upb_sink& sink) : sink_(sink) {} + Sink &operator=(const upb_sink &sink) { + sink_ = sink; + return *this; + } + + upb_sink sink() { return sink_; } + + /* Constructs a new sink for the given frozen handlers and closure. + * + * TODO: once the Handlers know the expected closure type, verify that T + * matches it. */ + template Sink(const upb_handlers* handlers, T* closure) { + Reset(handlers, closure); + } + + upb_sink* ptr() { return &sink_; } + + /* Resets the value of the sink. */ + template void Reset(const upb_handlers* handlers, T* closure) { + upb_sink_reset(&sink_, handlers, closure); + } + + /* Returns the top-level object that is bound to this sink. + * + * TODO: once the Handlers know the expected closure type, verify that T + * matches it. */ + template T* GetObject() const { + return static_cast(sink_.closure); + } + + /* Functions for pushing data into the sink. + * + * These return false if processing should stop (either due to error or just + * to suspend). + * + * These may not be called from within one of the same sink's handlers (in + * other words, handlers are not re-entrant). */ + + /* Should be called at the start and end of every message; both the top-level + * message and submessages. This means that submessages should use the + * following sequence: + * sink->StartSubMessage(startsubmsg_selector); + * sink->StartMessage(); + * // ... + * sink->EndMessage(&status); + * sink->EndSubMessage(endsubmsg_selector); */ + bool StartMessage() { return upb_sink_startmsg(sink_); } + bool EndMessage(upb_status *status) { + return upb_sink_endmsg(sink_, status); + } + + /* Putting of individual values. These work for both repeated and + * non-repeated fields, but for repeated fields you must wrap them in + * calls to StartSequence()/EndSequence(). */ + bool PutInt32(HandlersPtr::Selector s, int32_t val) { + return upb_sink_putint32(sink_, s, val); + } + + bool PutInt64(HandlersPtr::Selector s, int64_t val) { + return upb_sink_putint64(sink_, s, val); + } + + bool PutUInt32(HandlersPtr::Selector s, uint32_t val) { + return upb_sink_putuint32(sink_, s, val); + } + + bool PutUInt64(HandlersPtr::Selector s, uint64_t val) { + return upb_sink_putuint64(sink_, s, val); + } + + bool PutFloat(HandlersPtr::Selector s, float val) { + return upb_sink_putfloat(sink_, s, val); + } + + bool PutDouble(HandlersPtr::Selector s, double val) { + return upb_sink_putdouble(sink_, s, val); + } + + bool PutBool(HandlersPtr::Selector s, bool val) { + return upb_sink_putbool(sink_, s, val); + } + + /* Putting of string/bytes values. Each string can consist of zero or more + * non-contiguous buffers of data. + * + * For StartString(), the function will write a sink for the string to "sub." + * The sub-sink must be used for any/all PutStringBuffer() calls. */ + bool StartString(HandlersPtr::Selector s, size_t size_hint, Sink* sub) { + upb_sink sub_c; + bool ret = upb_sink_startstr(sink_, s, size_hint, &sub_c); + *sub = sub_c; + return ret; + } + + size_t PutStringBuffer(HandlersPtr::Selector s, const char *buf, size_t len, + const upb_bufhandle *handle) { + return upb_sink_putstring(sink_, s, buf, len, handle); + } + + bool EndString(HandlersPtr::Selector s) { + return upb_sink_endstr(sink_, s); + } + + /* For submessage fields. + * + * For StartSubMessage(), the function will write a sink for the string to + * "sub." The sub-sink must be used for any/all handlers called within the + * submessage. */ + bool StartSubMessage(HandlersPtr::Selector s, Sink* sub) { + upb_sink sub_c; + bool ret = upb_sink_startsubmsg(sink_, s, &sub_c); + *sub = sub_c; + return ret; + } + + bool EndSubMessage(HandlersPtr::Selector s) { + return upb_sink_endsubmsg(sink_, s); + } + + /* For repeated fields of any type, the sequence of values must be wrapped in + * these calls. + * + * For StartSequence(), the function will write a sink for the string to + * "sub." The sub-sink must be used for any/all handlers called within the + * sequence. */ + bool StartSequence(HandlersPtr::Selector s, Sink* sub) { + upb_sink sub_c; + bool ret = upb_sink_startseq(sink_, s, &sub_c); + *sub = sub_c; + return ret; + } + + bool EndSequence(HandlersPtr::Selector s) { + return upb_sink_endseq(sink_, s); + } + + /* Copy and assign specifically allowed. + * We don't even bother making these members private because so many + * functions need them and this is mainly just a dumb data container anyway. + */ + + private: + upb_sink sink_; +}; + +#endif /* __cplusplus */ + +/* upb_bytessink **************************************************************/ + +typedef struct { + const upb_byteshandler *handler; + void *closure; +} upb_bytessink ; + +UPB_INLINE void upb_bytessink_reset(upb_bytessink* s, const upb_byteshandler *h, + void *closure) { + s->handler = h; + s->closure = closure; +} + +UPB_INLINE bool upb_bytessink_start(upb_bytessink s, size_t size_hint, + void **subc) { + typedef upb_startstr_handlerfunc func; + func *start; + *subc = s.closure; + if (!s.handler) return true; + start = (func *)s.handler->table[UPB_STARTSTR_SELECTOR].func; + + if (!start) return true; + *subc = start(s.closure, + s.handler->table[UPB_STARTSTR_SELECTOR].attr.handler_data, + size_hint); + return *subc != NULL; +} + +UPB_INLINE size_t upb_bytessink_putbuf(upb_bytessink s, void *subc, + const char *buf, size_t size, + const upb_bufhandle* handle) { + typedef upb_string_handlerfunc func; + func *putbuf; + if (!s.handler) return true; + putbuf = (func *)s.handler->table[UPB_STRING_SELECTOR].func; + + if (!putbuf) return true; + return putbuf(subc, s.handler->table[UPB_STRING_SELECTOR].attr.handler_data, + buf, size, handle); +} + +UPB_INLINE bool upb_bytessink_end(upb_bytessink s) { + typedef upb_endfield_handlerfunc func; + func *end; + if (!s.handler) return true; + end = (func *)s.handler->table[UPB_ENDSTR_SELECTOR].func; + + if (!end) return true; + return end(s.closure, + s.handler->table[UPB_ENDSTR_SELECTOR].attr.handler_data); +} + +#ifdef __cplusplus + +class upb::BytesSink { + public: + BytesSink() {} + + BytesSink(const BytesSink&) = default; + BytesSink& operator=(const BytesSink&) = default; + + BytesSink(const upb_bytessink& sink) : sink_(sink) {} + BytesSink &operator=(const upb_bytessink &sink) { + sink_ = sink; + return *this; + } + + upb_bytessink sink() { return sink_; } + + /* Constructs a new sink for the given frozen handlers and closure. + * + * TODO(haberman): once the Handlers know the expected closure type, verify + * that T matches it. */ + template BytesSink(const upb_byteshandler* handler, T* closure) { + upb_bytessink_reset(sink_, handler, closure); + } + + /* Resets the value of the sink. */ + template void Reset(const upb_byteshandler* handler, T* closure) { + upb_bytessink_reset(&sink_, handler, closure); + } + + bool Start(size_t size_hint, void **subc) { + return upb_bytessink_start(sink_, size_hint, subc); + } + + size_t PutBuffer(void *subc, const char *buf, size_t len, + const upb_bufhandle *handle) { + return upb_bytessink_putbuf(sink_, subc, buf, len, handle); + } + + bool End() { + return upb_bytessink_end(sink_); + } + + private: + upb_bytessink sink_; +}; + +#endif /* __cplusplus */ + +/* upb_bufsrc *****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +bool upb_bufsrc_putbuf(const char *buf, size_t len, upb_bytessink sink); + +#ifdef __cplusplus +} /* extern "C" */ + +namespace upb { +template bool PutBuffer(const T& str, BytesSink sink) { + return upb_bufsrc_putbuf(str.data(), str.size(), sink.sink()); +} +} + +#endif /* __cplusplus */ + +#include "upb/port_undef.inc" + +#endif diff --git a/third_party/upb/upb/table.c b/third_party/upb/upb/table.c new file mode 100644 index 00000000000..8896d217db6 --- /dev/null +++ b/third_party/upb/upb/table.c @@ -0,0 +1,912 @@ +/* +** upb_table Implementation +** +** Implementation is heavily inspired by Lua's ltable.c. +*/ + +#include "upb/table.int.h" + +#include + +#include "upb/port_def.inc" + +#define UPB_MAXARRSIZE 16 /* 64k. */ + +/* From Chromium. */ +#define ARRAY_SIZE(x) \ + ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x]))))) + +static void upb_check_alloc(upb_table *t, upb_alloc *a) { + UPB_UNUSED(t); + UPB_UNUSED(a); + UPB_ASSERT_DEBUGVAR(t->alloc == a); +} + +static const double MAX_LOAD = 0.85; + +/* The minimum utilization of the array part of a mixed hash/array table. This + * is a speed/memory-usage tradeoff (though it's not straightforward because of + * cache effects). The lower this is, the more memory we'll use. */ +static const double MIN_DENSITY = 0.1; + +bool is_pow2(uint64_t v) { return v == 0 || (v & (v - 1)) == 0; } + +int log2ceil(uint64_t v) { + int ret = 0; + bool pow2 = is_pow2(v); + while (v >>= 1) ret++; + ret = pow2 ? ret : ret + 1; /* Ceiling. */ + return UPB_MIN(UPB_MAXARRSIZE, ret); +} + +char *upb_strdup(const char *s, upb_alloc *a) { + return upb_strdup2(s, strlen(s), a); +} + +char *upb_strdup2(const char *s, size_t len, upb_alloc *a) { + size_t n; + char *p; + + /* Prevent overflow errors. */ + if (len == SIZE_MAX) return NULL; + /* Always null-terminate, even if binary data; but don't rely on the input to + * have a null-terminating byte since it may be a raw binary buffer. */ + n = len + 1; + p = upb_malloc(a, n); + if (p) { + memcpy(p, s, len); + p[len] = 0; + } + return p; +} + +/* A type to represent the lookup key of either a strtable or an inttable. */ +typedef union { + uintptr_t num; + struct { + const char *str; + size_t len; + } str; +} lookupkey_t; + +static lookupkey_t strkey2(const char *str, size_t len) { + lookupkey_t k; + k.str.str = str; + k.str.len = len; + return k; +} + +static lookupkey_t intkey(uintptr_t key) { + lookupkey_t k; + k.num = key; + return k; +} + +typedef uint32_t hashfunc_t(upb_tabkey key); +typedef bool eqlfunc_t(upb_tabkey k1, lookupkey_t k2); + +/* Base table (shared code) ***************************************************/ + +/* For when we need to cast away const. */ +static upb_tabent *mutable_entries(upb_table *t) { + return (upb_tabent*)t->entries; +} + +static bool isfull(upb_table *t) { + if (upb_table_size(t) == 0) { + return true; + } else { + return ((double)(t->count + 1) / upb_table_size(t)) > MAX_LOAD; + } +} + +static bool init(upb_table *t, upb_ctype_t ctype, uint8_t size_lg2, + upb_alloc *a) { + size_t bytes; + + t->count = 0; + t->ctype = ctype; + t->size_lg2 = size_lg2; + t->mask = upb_table_size(t) ? upb_table_size(t) - 1 : 0; +#ifndef NDEBUG + t->alloc = a; +#endif + bytes = upb_table_size(t) * sizeof(upb_tabent); + if (bytes > 0) { + t->entries = upb_malloc(a, bytes); + if (!t->entries) return false; + memset(mutable_entries(t), 0, bytes); + } else { + t->entries = NULL; + } + return true; +} + +static void uninit(upb_table *t, upb_alloc *a) { + upb_check_alloc(t, a); + upb_free(a, mutable_entries(t)); +} + +static upb_tabent *emptyent(upb_table *t) { + upb_tabent *e = mutable_entries(t) + upb_table_size(t); + while (1) { if (upb_tabent_isempty(--e)) return e; UPB_ASSERT(e > t->entries); } +} + +static upb_tabent *getentry_mutable(upb_table *t, uint32_t hash) { + return (upb_tabent*)upb_getentry(t, hash); +} + +static const upb_tabent *findentry(const upb_table *t, lookupkey_t key, + uint32_t hash, eqlfunc_t *eql) { + const upb_tabent *e; + + if (t->size_lg2 == 0) return NULL; + e = upb_getentry(t, hash); + if (upb_tabent_isempty(e)) return NULL; + while (1) { + if (eql(e->key, key)) return e; + if ((e = e->next) == NULL) return NULL; + } +} + +static upb_tabent *findentry_mutable(upb_table *t, lookupkey_t key, + uint32_t hash, eqlfunc_t *eql) { + return (upb_tabent*)findentry(t, key, hash, eql); +} + +static bool lookup(const upb_table *t, lookupkey_t key, upb_value *v, + uint32_t hash, eqlfunc_t *eql) { + const upb_tabent *e = findentry(t, key, hash, eql); + if (e) { + if (v) { + _upb_value_setval(v, e->val.val, t->ctype); + } + return true; + } else { + return false; + } +} + +/* The given key must not already exist in the table. */ +static void insert(upb_table *t, lookupkey_t key, upb_tabkey tabkey, + upb_value val, uint32_t hash, + hashfunc_t *hashfunc, eqlfunc_t *eql) { + upb_tabent *mainpos_e; + upb_tabent *our_e; + + UPB_ASSERT(findentry(t, key, hash, eql) == NULL); + UPB_ASSERT_DEBUGVAR(val.ctype == t->ctype); + + t->count++; + mainpos_e = getentry_mutable(t, hash); + our_e = mainpos_e; + + if (upb_tabent_isempty(mainpos_e)) { + /* Our main position is empty; use it. */ + our_e->next = NULL; + } else { + /* Collision. */ + upb_tabent *new_e = emptyent(t); + /* Head of collider's chain. */ + upb_tabent *chain = getentry_mutable(t, hashfunc(mainpos_e->key)); + if (chain == mainpos_e) { + /* Existing ent is in its main posisiton (it has the same hash as us, and + * is the head of our chain). Insert to new ent and append to this chain. */ + new_e->next = mainpos_e->next; + mainpos_e->next = new_e; + our_e = new_e; + } else { + /* Existing ent is not in its main position (it is a node in some other + * chain). This implies that no existing ent in the table has our hash. + * Evict it (updating its chain) and use its ent for head of our chain. */ + *new_e = *mainpos_e; /* copies next. */ + while (chain->next != mainpos_e) { + chain = (upb_tabent*)chain->next; + UPB_ASSERT(chain); + } + chain->next = new_e; + our_e = mainpos_e; + our_e->next = NULL; + } + } + our_e->key = tabkey; + our_e->val.val = val.val; + UPB_ASSERT(findentry(t, key, hash, eql) == our_e); +} + +static bool rm(upb_table *t, lookupkey_t key, upb_value *val, + upb_tabkey *removed, uint32_t hash, eqlfunc_t *eql) { + upb_tabent *chain = getentry_mutable(t, hash); + if (upb_tabent_isempty(chain)) return false; + if (eql(chain->key, key)) { + /* Element to remove is at the head of its chain. */ + t->count--; + if (val) _upb_value_setval(val, chain->val.val, t->ctype); + if (removed) *removed = chain->key; + if (chain->next) { + upb_tabent *move = (upb_tabent*)chain->next; + *chain = *move; + move->key = 0; /* Make the slot empty. */ + } else { + chain->key = 0; /* Make the slot empty. */ + } + return true; + } else { + /* Element to remove is either in a non-head position or not in the + * table. */ + while (chain->next && !eql(chain->next->key, key)) { + chain = (upb_tabent*)chain->next; + } + if (chain->next) { + /* Found element to remove. */ + upb_tabent *rm = (upb_tabent*)chain->next; + t->count--; + if (val) _upb_value_setval(val, chain->next->val.val, t->ctype); + if (removed) *removed = rm->key; + rm->key = 0; /* Make the slot empty. */ + chain->next = rm->next; + return true; + } else { + /* Element to remove is not in the table. */ + return false; + } + } +} + +static size_t next(const upb_table *t, size_t i) { + do { + if (++i >= upb_table_size(t)) + return SIZE_MAX; + } while(upb_tabent_isempty(&t->entries[i])); + + return i; +} + +static size_t begin(const upb_table *t) { + return next(t, -1); +} + + +/* upb_strtable ***************************************************************/ + +/* A simple "subclass" of upb_table that only adds a hash function for strings. */ + +static upb_tabkey strcopy(lookupkey_t k2, upb_alloc *a) { + uint32_t len = (uint32_t) k2.str.len; + char *str = upb_malloc(a, k2.str.len + sizeof(uint32_t) + 1); + if (str == NULL) return 0; + memcpy(str, &len, sizeof(uint32_t)); + memcpy(str + sizeof(uint32_t), k2.str.str, k2.str.len); + str[sizeof(uint32_t) + k2.str.len] = '\0'; + return (uintptr_t)str; +} + +static uint32_t strhash(upb_tabkey key) { + uint32_t len; + char *str = upb_tabstr(key, &len); + return upb_murmur_hash2(str, len, 0); +} + +static bool streql(upb_tabkey k1, lookupkey_t k2) { + uint32_t len; + char *str = upb_tabstr(k1, &len); + return len == k2.str.len && memcmp(str, k2.str.str, len) == 0; +} + +bool upb_strtable_init2(upb_strtable *t, upb_ctype_t ctype, upb_alloc *a) { + return init(&t->t, ctype, 2, a); +} + +void upb_strtable_uninit2(upb_strtable *t, upb_alloc *a) { + size_t i; + for (i = 0; i < upb_table_size(&t->t); i++) + upb_free(a, (void*)t->t.entries[i].key); + uninit(&t->t, a); +} + +bool upb_strtable_resize(upb_strtable *t, size_t size_lg2, upb_alloc *a) { + upb_strtable new_table; + upb_strtable_iter i; + + upb_check_alloc(&t->t, a); + + if (!init(&new_table.t, t->t.ctype, size_lg2, a)) + return false; + upb_strtable_begin(&i, t); + for ( ; !upb_strtable_done(&i); upb_strtable_next(&i)) { + upb_strtable_insert3( + &new_table, + upb_strtable_iter_key(&i), + upb_strtable_iter_keylength(&i), + upb_strtable_iter_value(&i), + a); + } + upb_strtable_uninit2(t, a); + *t = new_table; + return true; +} + +bool upb_strtable_insert3(upb_strtable *t, const char *k, size_t len, + upb_value v, upb_alloc *a) { + lookupkey_t key; + upb_tabkey tabkey; + uint32_t hash; + + upb_check_alloc(&t->t, a); + + if (isfull(&t->t)) { + /* Need to resize. New table of double the size, add old elements to it. */ + if (!upb_strtable_resize(t, t->t.size_lg2 + 1, a)) { + return false; + } + } + + key = strkey2(k, len); + tabkey = strcopy(key, a); + if (tabkey == 0) return false; + + hash = upb_murmur_hash2(key.str.str, key.str.len, 0); + insert(&t->t, key, tabkey, v, hash, &strhash, &streql); + return true; +} + +bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len, + upb_value *v) { + uint32_t hash = upb_murmur_hash2(key, len, 0); + return lookup(&t->t, strkey2(key, len), v, hash, &streql); +} + +bool upb_strtable_remove3(upb_strtable *t, const char *key, size_t len, + upb_value *val, upb_alloc *alloc) { + uint32_t hash = upb_murmur_hash2(key, len, 0); + upb_tabkey tabkey; + if (rm(&t->t, strkey2(key, len), val, &tabkey, hash, &streql)) { + upb_free(alloc, (void*)tabkey); + return true; + } else { + return false; + } +} + +/* Iteration */ + +static const upb_tabent *str_tabent(const upb_strtable_iter *i) { + return &i->t->t.entries[i->index]; +} + +void upb_strtable_begin(upb_strtable_iter *i, const upb_strtable *t) { + i->t = t; + i->index = begin(&t->t); +} + +void upb_strtable_next(upb_strtable_iter *i) { + i->index = next(&i->t->t, i->index); +} + +bool upb_strtable_done(const upb_strtable_iter *i) { + if (!i->t) return true; + return i->index >= upb_table_size(&i->t->t) || + upb_tabent_isempty(str_tabent(i)); +} + +const char *upb_strtable_iter_key(const upb_strtable_iter *i) { + UPB_ASSERT(!upb_strtable_done(i)); + return upb_tabstr(str_tabent(i)->key, NULL); +} + +size_t upb_strtable_iter_keylength(const upb_strtable_iter *i) { + uint32_t len; + UPB_ASSERT(!upb_strtable_done(i)); + upb_tabstr(str_tabent(i)->key, &len); + return len; +} + +upb_value upb_strtable_iter_value(const upb_strtable_iter *i) { + UPB_ASSERT(!upb_strtable_done(i)); + return _upb_value_val(str_tabent(i)->val.val, i->t->t.ctype); +} + +void upb_strtable_iter_setdone(upb_strtable_iter *i) { + i->t = NULL; + i->index = SIZE_MAX; +} + +bool upb_strtable_iter_isequal(const upb_strtable_iter *i1, + const upb_strtable_iter *i2) { + if (upb_strtable_done(i1) && upb_strtable_done(i2)) + return true; + return i1->t == i2->t && i1->index == i2->index; +} + + +/* upb_inttable ***************************************************************/ + +/* For inttables we use a hybrid structure where small keys are kept in an + * array and large keys are put in the hash table. */ + +static uint32_t inthash(upb_tabkey key) { return upb_inthash(key); } + +static bool inteql(upb_tabkey k1, lookupkey_t k2) { + return k1 == k2.num; +} + +static upb_tabval *mutable_array(upb_inttable *t) { + return (upb_tabval*)t->array; +} + +static upb_tabval *inttable_val(upb_inttable *t, uintptr_t key) { + if (key < t->array_size) { + return upb_arrhas(t->array[key]) ? &(mutable_array(t)[key]) : NULL; + } else { + upb_tabent *e = + findentry_mutable(&t->t, intkey(key), upb_inthash(key), &inteql); + return e ? &e->val : NULL; + } +} + +static const upb_tabval *inttable_val_const(const upb_inttable *t, + uintptr_t key) { + return inttable_val((upb_inttable*)t, key); +} + +size_t upb_inttable_count(const upb_inttable *t) { + return t->t.count + t->array_count; +} + +static void check(upb_inttable *t) { + UPB_UNUSED(t); +#if defined(UPB_DEBUG_TABLE) && !defined(NDEBUG) + { + /* This check is very expensive (makes inserts/deletes O(N)). */ + size_t count = 0; + upb_inttable_iter i; + upb_inttable_begin(&i, t); + for(; !upb_inttable_done(&i); upb_inttable_next(&i), count++) { + UPB_ASSERT(upb_inttable_lookup(t, upb_inttable_iter_key(&i), NULL)); + } + UPB_ASSERT(count == upb_inttable_count(t)); + } +#endif +} + +bool upb_inttable_sizedinit(upb_inttable *t, upb_ctype_t ctype, + size_t asize, int hsize_lg2, upb_alloc *a) { + size_t array_bytes; + + if (!init(&t->t, ctype, hsize_lg2, a)) return false; + /* Always make the array part at least 1 long, so that we know key 0 + * won't be in the hash part, which simplifies things. */ + t->array_size = UPB_MAX(1, asize); + t->array_count = 0; + array_bytes = t->array_size * sizeof(upb_value); + t->array = upb_malloc(a, array_bytes); + if (!t->array) { + uninit(&t->t, a); + return false; + } + memset(mutable_array(t), 0xff, array_bytes); + check(t); + return true; +} + +bool upb_inttable_init2(upb_inttable *t, upb_ctype_t ctype, upb_alloc *a) { + return upb_inttable_sizedinit(t, ctype, 0, 4, a); +} + +void upb_inttable_uninit2(upb_inttable *t, upb_alloc *a) { + uninit(&t->t, a); + upb_free(a, mutable_array(t)); +} + +bool upb_inttable_insert2(upb_inttable *t, uintptr_t key, upb_value val, + upb_alloc *a) { + upb_tabval tabval; + tabval.val = val.val; + UPB_ASSERT(upb_arrhas(tabval)); /* This will reject (uint64_t)-1. Fix this. */ + + upb_check_alloc(&t->t, a); + + if (key < t->array_size) { + UPB_ASSERT(!upb_arrhas(t->array[key])); + t->array_count++; + mutable_array(t)[key].val = val.val; + } else { + if (isfull(&t->t)) { + /* Need to resize the hash part, but we re-use the array part. */ + size_t i; + upb_table new_table; + + if (!init(&new_table, t->t.ctype, t->t.size_lg2 + 1, a)) { + return false; + } + + for (i = begin(&t->t); i < upb_table_size(&t->t); i = next(&t->t, i)) { + const upb_tabent *e = &t->t.entries[i]; + uint32_t hash; + upb_value v; + + _upb_value_setval(&v, e->val.val, t->t.ctype); + hash = upb_inthash(e->key); + insert(&new_table, intkey(e->key), e->key, v, hash, &inthash, &inteql); + } + + UPB_ASSERT(t->t.count == new_table.count); + + uninit(&t->t, a); + t->t = new_table; + } + insert(&t->t, intkey(key), key, val, upb_inthash(key), &inthash, &inteql); + } + check(t); + return true; +} + +bool upb_inttable_lookup(const upb_inttable *t, uintptr_t key, upb_value *v) { + const upb_tabval *table_v = inttable_val_const(t, key); + if (!table_v) return false; + if (v) _upb_value_setval(v, table_v->val, t->t.ctype); + return true; +} + +bool upb_inttable_replace(upb_inttable *t, uintptr_t key, upb_value val) { + upb_tabval *table_v = inttable_val(t, key); + if (!table_v) return false; + table_v->val = val.val; + return true; +} + +bool upb_inttable_remove(upb_inttable *t, uintptr_t key, upb_value *val) { + bool success; + if (key < t->array_size) { + if (upb_arrhas(t->array[key])) { + upb_tabval empty = UPB_TABVALUE_EMPTY_INIT; + t->array_count--; + if (val) { + _upb_value_setval(val, t->array[key].val, t->t.ctype); + } + mutable_array(t)[key] = empty; + success = true; + } else { + success = false; + } + } else { + success = rm(&t->t, intkey(key), val, NULL, upb_inthash(key), &inteql); + } + check(t); + return success; +} + +bool upb_inttable_push2(upb_inttable *t, upb_value val, upb_alloc *a) { + upb_check_alloc(&t->t, a); + return upb_inttable_insert2(t, upb_inttable_count(t), val, a); +} + +upb_value upb_inttable_pop(upb_inttable *t) { + upb_value val; + bool ok = upb_inttable_remove(t, upb_inttable_count(t) - 1, &val); + UPB_ASSERT(ok); + return val; +} + +bool upb_inttable_insertptr2(upb_inttable *t, const void *key, upb_value val, + upb_alloc *a) { + upb_check_alloc(&t->t, a); + return upb_inttable_insert2(t, (uintptr_t)key, val, a); +} + +bool upb_inttable_lookupptr(const upb_inttable *t, const void *key, + upb_value *v) { + return upb_inttable_lookup(t, (uintptr_t)key, v); +} + +bool upb_inttable_removeptr(upb_inttable *t, const void *key, upb_value *val) { + return upb_inttable_remove(t, (uintptr_t)key, val); +} + +void upb_inttable_compact2(upb_inttable *t, upb_alloc *a) { + /* A power-of-two histogram of the table keys. */ + size_t counts[UPB_MAXARRSIZE + 1] = {0}; + + /* The max key in each bucket. */ + uintptr_t max[UPB_MAXARRSIZE + 1] = {0}; + + upb_inttable_iter i; + size_t arr_count; + int size_lg2; + upb_inttable new_t; + + upb_check_alloc(&t->t, a); + + upb_inttable_begin(&i, t); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + uintptr_t key = upb_inttable_iter_key(&i); + int bucket = log2ceil(key); + max[bucket] = UPB_MAX(max[bucket], key); + counts[bucket]++; + } + + /* Find the largest power of two that satisfies the MIN_DENSITY + * definition (while actually having some keys). */ + arr_count = upb_inttable_count(t); + + for (size_lg2 = ARRAY_SIZE(counts) - 1; size_lg2 > 0; size_lg2--) { + if (counts[size_lg2] == 0) { + /* We can halve again without losing any entries. */ + continue; + } else if (arr_count >= (1 << size_lg2) * MIN_DENSITY) { + break; + } + + arr_count -= counts[size_lg2]; + } + + UPB_ASSERT(arr_count <= upb_inttable_count(t)); + + { + /* Insert all elements into new, perfectly-sized table. */ + size_t arr_size = max[size_lg2] + 1; /* +1 so arr[max] will fit. */ + size_t hash_count = upb_inttable_count(t) - arr_count; + size_t hash_size = hash_count ? (hash_count / MAX_LOAD) + 1 : 0; + int hashsize_lg2 = log2ceil(hash_size); + + upb_inttable_sizedinit(&new_t, t->t.ctype, arr_size, hashsize_lg2, a); + upb_inttable_begin(&i, t); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + uintptr_t k = upb_inttable_iter_key(&i); + upb_inttable_insert2(&new_t, k, upb_inttable_iter_value(&i), a); + } + UPB_ASSERT(new_t.array_size == arr_size); + UPB_ASSERT(new_t.t.size_lg2 == hashsize_lg2); + } + upb_inttable_uninit2(t, a); + *t = new_t; +} + +/* Iteration. */ + +static const upb_tabent *int_tabent(const upb_inttable_iter *i) { + UPB_ASSERT(!i->array_part); + return &i->t->t.entries[i->index]; +} + +static upb_tabval int_arrent(const upb_inttable_iter *i) { + UPB_ASSERT(i->array_part); + return i->t->array[i->index]; +} + +void upb_inttable_begin(upb_inttable_iter *i, const upb_inttable *t) { + i->t = t; + i->index = -1; + i->array_part = true; + upb_inttable_next(i); +} + +void upb_inttable_next(upb_inttable_iter *iter) { + const upb_inttable *t = iter->t; + if (iter->array_part) { + while (++iter->index < t->array_size) { + if (upb_arrhas(int_arrent(iter))) { + return; + } + } + iter->array_part = false; + iter->index = begin(&t->t); + } else { + iter->index = next(&t->t, iter->index); + } +} + +bool upb_inttable_done(const upb_inttable_iter *i) { + if (!i->t) return true; + if (i->array_part) { + return i->index >= i->t->array_size || + !upb_arrhas(int_arrent(i)); + } else { + return i->index >= upb_table_size(&i->t->t) || + upb_tabent_isempty(int_tabent(i)); + } +} + +uintptr_t upb_inttable_iter_key(const upb_inttable_iter *i) { + UPB_ASSERT(!upb_inttable_done(i)); + return i->array_part ? i->index : int_tabent(i)->key; +} + +upb_value upb_inttable_iter_value(const upb_inttable_iter *i) { + UPB_ASSERT(!upb_inttable_done(i)); + return _upb_value_val( + i->array_part ? i->t->array[i->index].val : int_tabent(i)->val.val, + i->t->t.ctype); +} + +void upb_inttable_iter_setdone(upb_inttable_iter *i) { + i->t = NULL; + i->index = SIZE_MAX; + i->array_part = false; +} + +bool upb_inttable_iter_isequal(const upb_inttable_iter *i1, + const upb_inttable_iter *i2) { + if (upb_inttable_done(i1) && upb_inttable_done(i2)) + return true; + return i1->t == i2->t && i1->index == i2->index && + i1->array_part == i2->array_part; +} + +#if defined(UPB_UNALIGNED_READS_OK) || defined(__s390x__) +/* ----------------------------------------------------------------------------- + * MurmurHash2, by Austin Appleby (released as public domain). + * Reformatted and C99-ified by Joshua Haberman. + * Note - This code makes a few assumptions about how your machine behaves - + * 1. We can read a 4-byte value from any address without crashing + * 2. sizeof(int) == 4 (in upb this limitation is removed by using uint32_t + * And it has a few limitations - + * 1. It will not work incrementally. + * 2. It will not produce the same results on little-endian and big-endian + * machines. */ +uint32_t upb_murmur_hash2(const void *key, size_t len, uint32_t seed) { + /* 'm' and 'r' are mixing constants generated offline. + * They're not really 'magic', they just happen to work well. */ + const uint32_t m = 0x5bd1e995; + const int32_t r = 24; + + /* Initialize the hash to a 'random' value */ + uint32_t h = seed ^ len; + + /* Mix 4 bytes at a time into the hash */ + const uint8_t * data = (const uint8_t *)key; + while(len >= 4) { + uint32_t k = *(uint32_t *)data; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + /* Handle the last few bytes of the input array */ + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + + /* Do a few final mixes of the hash to ensure the last few + * bytes are well-incorporated. */ + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +#else /* !UPB_UNALIGNED_READS_OK */ + +/* ----------------------------------------------------------------------------- + * MurmurHashAligned2, by Austin Appleby + * Same algorithm as MurmurHash2, but only does aligned reads - should be safer + * on certain platforms. + * Performance will be lower than MurmurHash2 */ + +#define MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; } + +uint32_t upb_murmur_hash2(const void * key, size_t len, uint32_t seed) { + const uint32_t m = 0x5bd1e995; + const int32_t r = 24; + const uint8_t * data = (const uint8_t *)key; + uint32_t h = (uint32_t)(seed ^ len); + uint8_t align = (uintptr_t)data & 3; + + if(align && (len >= 4)) { + /* Pre-load the temp registers */ + uint32_t t = 0, d = 0; + int32_t sl; + int32_t sr; + + switch(align) { + case 1: t |= data[2] << 16; + case 2: t |= data[1] << 8; + case 3: t |= data[0]; + } + + t <<= (8 * align); + + data += 4-align; + len -= 4-align; + + sl = 8 * (4-align); + sr = 8 * align; + + /* Mix */ + + while(len >= 4) { + uint32_t k; + + d = *(uint32_t *)data; + t = (t >> sr) | (d << sl); + + k = t; + + MIX(h,k,m); + + t = d; + + data += 4; + len -= 4; + } + + /* Handle leftover data in temp registers */ + + d = 0; + + if(len >= align) { + uint32_t k; + + switch(align) { + case 3: d |= data[2] << 16; + case 2: d |= data[1] << 8; + case 1: d |= data[0]; + } + + k = (t >> sr) | (d << sl); + MIX(h,k,m); + + data += align; + len -= align; + + /* ---------- + * Handle tail bytes */ + + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + } else { + switch(len) { + case 3: d |= data[2] << 16; + case 2: d |= data[1] << 8; + case 1: d |= data[0]; + case 0: h ^= (t >> sr) | (d << sl); h *= m; + } + } + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; + } else { + while(len >= 4) { + uint32_t k = *(uint32_t *)data; + + MIX(h,k,m); + + data += 4; + len -= 4; + } + + /* ---------- + * Handle tail bytes */ + + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; + } +} +#undef MIX + +#endif /* UPB_UNALIGNED_READS_OK */ diff --git a/third_party/upb/upb/table.int.h b/third_party/upb/upb/table.int.h new file mode 100644 index 00000000000..23b0b2f2213 --- /dev/null +++ b/third_party/upb/upb/table.int.h @@ -0,0 +1,507 @@ +/* +** upb_table +** +** This header is INTERNAL-ONLY! Its interfaces are not public or stable! +** This file defines very fast int->upb_value (inttable) and string->upb_value +** (strtable) hash tables. +** +** The table uses chained scatter with Brent's variation (inspired by the Lua +** implementation of hash tables). The hash function for strings is Austin +** Appleby's "MurmurHash." +** +** The inttable uses uintptr_t as its key, which guarantees it can be used to +** store pointers or integers of at least 32 bits (upb isn't really useful on +** systems where sizeof(void*) < 4). +** +** The table must be homogenous (all values of the same type). In debug +** mode, we check this on insert and lookup. +*/ + +#ifndef UPB_TABLE_H_ +#define UPB_TABLE_H_ + +#include +#include +#include "upb/upb.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + + +/* upb_value ******************************************************************/ + +/* A tagged union (stored untagged inside the table) so that we can check that + * clients calling table accessors are correctly typed without having to have + * an explosion of accessors. */ +typedef enum { + UPB_CTYPE_INT32 = 1, + UPB_CTYPE_INT64 = 2, + UPB_CTYPE_UINT32 = 3, + UPB_CTYPE_UINT64 = 4, + UPB_CTYPE_BOOL = 5, + UPB_CTYPE_CSTR = 6, + UPB_CTYPE_PTR = 7, + UPB_CTYPE_CONSTPTR = 8, + UPB_CTYPE_FPTR = 9, + UPB_CTYPE_FLOAT = 10, + UPB_CTYPE_DOUBLE = 11 +} upb_ctype_t; + +typedef struct { + uint64_t val; +#ifndef NDEBUG + /* In debug mode we carry the value type around also so we can check accesses + * to be sure the right member is being read. */ + upb_ctype_t ctype; +#endif +} upb_value; + +#ifdef NDEBUG +#define SET_TYPE(dest, val) UPB_UNUSED(val) +#else +#define SET_TYPE(dest, val) dest = val +#endif + +/* Like strdup(), which isn't always available since it's not ANSI C. */ +char *upb_strdup(const char *s, upb_alloc *a); +/* Variant that works with a length-delimited rather than NULL-delimited string, + * as supported by strtable. */ +char *upb_strdup2(const char *s, size_t len, upb_alloc *a); + +UPB_INLINE char *upb_gstrdup(const char *s) { + return upb_strdup(s, &upb_alloc_global); +} + +UPB_INLINE void _upb_value_setval(upb_value *v, uint64_t val, + upb_ctype_t ctype) { + v->val = val; + SET_TYPE(v->ctype, ctype); +} + +UPB_INLINE upb_value _upb_value_val(uint64_t val, upb_ctype_t ctype) { + upb_value ret; + _upb_value_setval(&ret, val, ctype); + return ret; +} + +/* For each value ctype, define the following set of functions: + * + * // Get/set an int32 from a upb_value. + * int32_t upb_value_getint32(upb_value val); + * void upb_value_setint32(upb_value *val, int32_t cval); + * + * // Construct a new upb_value from an int32. + * upb_value upb_value_int32(int32_t val); */ +#define FUNCS(name, membername, type_t, converter, proto_type) \ + UPB_INLINE void upb_value_set ## name(upb_value *val, type_t cval) { \ + val->val = (converter)cval; \ + SET_TYPE(val->ctype, proto_type); \ + } \ + UPB_INLINE upb_value upb_value_ ## name(type_t val) { \ + upb_value ret; \ + upb_value_set ## name(&ret, val); \ + return ret; \ + } \ + UPB_INLINE type_t upb_value_get ## name(upb_value val) { \ + UPB_ASSERT_DEBUGVAR(val.ctype == proto_type); \ + return (type_t)(converter)val.val; \ + } + +FUNCS(int32, int32, int32_t, int32_t, UPB_CTYPE_INT32) +FUNCS(int64, int64, int64_t, int64_t, UPB_CTYPE_INT64) +FUNCS(uint32, uint32, uint32_t, uint32_t, UPB_CTYPE_UINT32) +FUNCS(uint64, uint64, uint64_t, uint64_t, UPB_CTYPE_UINT64) +FUNCS(bool, _bool, bool, bool, UPB_CTYPE_BOOL) +FUNCS(cstr, cstr, char*, uintptr_t, UPB_CTYPE_CSTR) +FUNCS(ptr, ptr, void*, uintptr_t, UPB_CTYPE_PTR) +FUNCS(constptr, constptr, const void*, uintptr_t, UPB_CTYPE_CONSTPTR) +FUNCS(fptr, fptr, upb_func*, uintptr_t, UPB_CTYPE_FPTR) + +#undef FUNCS + +UPB_INLINE void upb_value_setfloat(upb_value *val, float cval) { + memcpy(&val->val, &cval, sizeof(cval)); + SET_TYPE(val->ctype, UPB_CTYPE_FLOAT); +} + +UPB_INLINE void upb_value_setdouble(upb_value *val, double cval) { + memcpy(&val->val, &cval, sizeof(cval)); + SET_TYPE(val->ctype, UPB_CTYPE_DOUBLE); +} + +UPB_INLINE upb_value upb_value_float(float cval) { + upb_value ret; + upb_value_setfloat(&ret, cval); + return ret; +} + +UPB_INLINE upb_value upb_value_double(double cval) { + upb_value ret; + upb_value_setdouble(&ret, cval); + return ret; +} + +#undef SET_TYPE + + +/* upb_tabkey *****************************************************************/ + +/* Either: + * 1. an actual integer key, or + * 2. a pointer to a string prefixed by its uint32_t length, owned by us. + * + * ...depending on whether this is a string table or an int table. We would + * make this a union of those two types, but C89 doesn't support statically + * initializing a non-first union member. */ +typedef uintptr_t upb_tabkey; + +UPB_INLINE char *upb_tabstr(upb_tabkey key, uint32_t *len) { + char* mem = (char*)key; + if (len) memcpy(len, mem, sizeof(*len)); + return mem + sizeof(*len); +} + + +/* upb_tabval *****************************************************************/ + +typedef struct { + uint64_t val; +} upb_tabval; + +#define UPB_TABVALUE_EMPTY_INIT {-1} + + +/* upb_table ******************************************************************/ + +typedef struct _upb_tabent { + upb_tabkey key; + upb_tabval val; + + /* Internal chaining. This is const so we can create static initializers for + * tables. We cast away const sometimes, but *only* when the containing + * upb_table is known to be non-const. This requires a bit of care, but + * the subtlety is confined to table.c. */ + const struct _upb_tabent *next; +} upb_tabent; + +typedef struct { + size_t count; /* Number of entries in the hash part. */ + size_t mask; /* Mask to turn hash value -> bucket. */ + upb_ctype_t ctype; /* Type of all values. */ + uint8_t size_lg2; /* Size of the hashtable part is 2^size_lg2 entries. */ + + /* Hash table entries. + * Making this const isn't entirely accurate; what we really want is for it to + * have the same const-ness as the table it's inside. But there's no way to + * declare that in C. So we have to make it const so that we can statically + * initialize const hash tables. Then we cast away const when we have to. + */ + const upb_tabent *entries; + +#ifndef NDEBUG + /* This table's allocator. We make the user pass it in to every relevant + * function and only use this to check it in debug mode. We do this solely + * to keep upb_table as small as possible. This might seem slightly paranoid + * but the plan is to use upb_table for all map fields and extension sets in + * a forthcoming message representation, so there could be a lot of these. + * If this turns out to be too annoying later, we can change it (since this + * is an internal-only header file). */ + upb_alloc *alloc; +#endif +} upb_table; + +typedef struct { + upb_table t; +} upb_strtable; + +typedef struct { + upb_table t; /* For entries that don't fit in the array part. */ + const upb_tabval *array; /* Array part of the table. See const note above. */ + size_t array_size; /* Array part size. */ + size_t array_count; /* Array part number of elements. */ +} upb_inttable; + +#define UPB_INTTABLE_INIT(count, mask, ctype, size_lg2, ent, a, asize, acount) \ + {UPB_TABLE_INIT(count, mask, ctype, size_lg2, ent), a, asize, acount} + +#define UPB_EMPTY_INTTABLE_INIT(ctype) \ + UPB_INTTABLE_INIT(0, 0, ctype, 0, NULL, NULL, 0, 0) + +#define UPB_ARRAY_EMPTYENT -1 + +UPB_INLINE size_t upb_table_size(const upb_table *t) { + if (t->size_lg2 == 0) + return 0; + else + return 1 << t->size_lg2; +} + +/* Internal-only functions, in .h file only out of necessity. */ +UPB_INLINE bool upb_tabent_isempty(const upb_tabent *e) { + return e->key == 0; +} + +/* Used by some of the unit tests for generic hashing functionality. */ +uint32_t upb_murmur_hash2(const void * key, size_t len, uint32_t seed); + +UPB_INLINE uintptr_t upb_intkey(uintptr_t key) { + return key; +} + +UPB_INLINE uint32_t upb_inthash(uintptr_t key) { + return (uint32_t)key; +} + +static const upb_tabent *upb_getentry(const upb_table *t, uint32_t hash) { + return t->entries + (hash & t->mask); +} + +UPB_INLINE bool upb_arrhas(upb_tabval key) { + return key.val != (uint64_t)-1; +} + +/* Initialize and uninitialize a table, respectively. If memory allocation + * failed, false is returned that the table is uninitialized. */ +bool upb_inttable_init2(upb_inttable *table, upb_ctype_t ctype, upb_alloc *a); +bool upb_strtable_init2(upb_strtable *table, upb_ctype_t ctype, upb_alloc *a); +void upb_inttable_uninit2(upb_inttable *table, upb_alloc *a); +void upb_strtable_uninit2(upb_strtable *table, upb_alloc *a); + +UPB_INLINE bool upb_inttable_init(upb_inttable *table, upb_ctype_t ctype) { + return upb_inttable_init2(table, ctype, &upb_alloc_global); +} + +UPB_INLINE bool upb_strtable_init(upb_strtable *table, upb_ctype_t ctype) { + return upb_strtable_init2(table, ctype, &upb_alloc_global); +} + +UPB_INLINE void upb_inttable_uninit(upb_inttable *table) { + upb_inttable_uninit2(table, &upb_alloc_global); +} + +UPB_INLINE void upb_strtable_uninit(upb_strtable *table) { + upb_strtable_uninit2(table, &upb_alloc_global); +} + +/* Returns the number of values in the table. */ +size_t upb_inttable_count(const upb_inttable *t); +UPB_INLINE size_t upb_strtable_count(const upb_strtable *t) { + return t->t.count; +} + +void upb_inttable_packedsize(const upb_inttable *t, size_t *size); +void upb_strtable_packedsize(const upb_strtable *t, size_t *size); +upb_inttable *upb_inttable_pack(const upb_inttable *t, void *p, size_t *ofs, + size_t size); +upb_strtable *upb_strtable_pack(const upb_strtable *t, void *p, size_t *ofs, + size_t size); + +/* Inserts the given key into the hashtable with the given value. The key must + * not already exist in the hash table. For string tables, the key must be + * NULL-terminated, and the table will make an internal copy of the key. + * Inttables must not insert a value of UINTPTR_MAX. + * + * If a table resize was required but memory allocation failed, false is + * returned and the table is unchanged. */ +bool upb_inttable_insert2(upb_inttable *t, uintptr_t key, upb_value val, + upb_alloc *a); +bool upb_strtable_insert3(upb_strtable *t, const char *key, size_t len, + upb_value val, upb_alloc *a); + +UPB_INLINE bool upb_inttable_insert(upb_inttable *t, uintptr_t key, + upb_value val) { + return upb_inttable_insert2(t, key, val, &upb_alloc_global); +} + +UPB_INLINE bool upb_strtable_insert2(upb_strtable *t, const char *key, + size_t len, upb_value val) { + return upb_strtable_insert3(t, key, len, val, &upb_alloc_global); +} + +/* For NULL-terminated strings. */ +UPB_INLINE bool upb_strtable_insert(upb_strtable *t, const char *key, + upb_value val) { + return upb_strtable_insert2(t, key, strlen(key), val); +} + +/* Looks up key in this table, returning "true" if the key was found. + * If v is non-NULL, copies the value for this key into *v. */ +bool upb_inttable_lookup(const upb_inttable *t, uintptr_t key, upb_value *v); +bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len, + upb_value *v); + +/* For NULL-terminated strings. */ +UPB_INLINE bool upb_strtable_lookup(const upb_strtable *t, const char *key, + upb_value *v) { + return upb_strtable_lookup2(t, key, strlen(key), v); +} + +/* Removes an item from the table. Returns true if the remove was successful, + * and stores the removed item in *val if non-NULL. */ +bool upb_inttable_remove(upb_inttable *t, uintptr_t key, upb_value *val); +bool upb_strtable_remove3(upb_strtable *t, const char *key, size_t len, + upb_value *val, upb_alloc *alloc); + +UPB_INLINE bool upb_strtable_remove2(upb_strtable *t, const char *key, + size_t len, upb_value *val) { + return upb_strtable_remove3(t, key, len, val, &upb_alloc_global); +} + +/* For NULL-terminated strings. */ +UPB_INLINE bool upb_strtable_remove(upb_strtable *t, const char *key, + upb_value *v) { + return upb_strtable_remove2(t, key, strlen(key), v); +} + +/* Updates an existing entry in an inttable. If the entry does not exist, + * returns false and does nothing. Unlike insert/remove, this does not + * invalidate iterators. */ +bool upb_inttable_replace(upb_inttable *t, uintptr_t key, upb_value val); + +/* Handy routines for treating an inttable like a stack. May not be mixed with + * other insert/remove calls. */ +bool upb_inttable_push2(upb_inttable *t, upb_value val, upb_alloc *a); +upb_value upb_inttable_pop(upb_inttable *t); + +UPB_INLINE bool upb_inttable_push(upb_inttable *t, upb_value val) { + return upb_inttable_push2(t, val, &upb_alloc_global); +} + +/* Convenience routines for inttables with pointer keys. */ +bool upb_inttable_insertptr2(upb_inttable *t, const void *key, upb_value val, + upb_alloc *a); +bool upb_inttable_removeptr(upb_inttable *t, const void *key, upb_value *val); +bool upb_inttable_lookupptr( + const upb_inttable *t, const void *key, upb_value *val); + +UPB_INLINE bool upb_inttable_insertptr(upb_inttable *t, const void *key, + upb_value val) { + return upb_inttable_insertptr2(t, key, val, &upb_alloc_global); +} + +/* Optimizes the table for the current set of entries, for both memory use and + * lookup time. Client should call this after all entries have been inserted; + * inserting more entries is legal, but will likely require a table resize. */ +void upb_inttable_compact2(upb_inttable *t, upb_alloc *a); + +UPB_INLINE void upb_inttable_compact(upb_inttable *t) { + upb_inttable_compact2(t, &upb_alloc_global); +} + +/* A special-case inlinable version of the lookup routine for 32-bit + * integers. */ +UPB_INLINE bool upb_inttable_lookup32(const upb_inttable *t, uint32_t key, + upb_value *v) { + *v = upb_value_int32(0); /* Silence compiler warnings. */ + if (key < t->array_size) { + upb_tabval arrval = t->array[key]; + if (upb_arrhas(arrval)) { + _upb_value_setval(v, arrval.val, t->t.ctype); + return true; + } else { + return false; + } + } else { + const upb_tabent *e; + if (t->t.entries == NULL) return false; + for (e = upb_getentry(&t->t, upb_inthash(key)); true; e = e->next) { + if ((uint32_t)e->key == key) { + _upb_value_setval(v, e->val.val, t->t.ctype); + return true; + } + if (e->next == NULL) return false; + } + } +} + +/* Exposed for testing only. */ +bool upb_strtable_resize(upb_strtable *t, size_t size_lg2, upb_alloc *a); + +/* Iterators ******************************************************************/ + +/* Iterators for int and string tables. We are subject to some kind of unusual + * design constraints: + * + * For high-level languages: + * - we must be able to guarantee that we don't crash or corrupt memory even if + * the program accesses an invalidated iterator. + * + * For C++11 range-based for: + * - iterators must be copyable + * - iterators must be comparable + * - it must be possible to construct an "end" value. + * + * Iteration order is undefined. + * + * Modifying the table invalidates iterators. upb_{str,int}table_done() is + * guaranteed to work even on an invalidated iterator, as long as the table it + * is iterating over has not been freed. Calling next() or accessing data from + * an invalidated iterator yields unspecified elements from the table, but it is + * guaranteed not to crash and to return real table elements (except when done() + * is true). */ + + +/* upb_strtable_iter **********************************************************/ + +/* upb_strtable_iter i; + * upb_strtable_begin(&i, t); + * for(; !upb_strtable_done(&i); upb_strtable_next(&i)) { + * const char *key = upb_strtable_iter_key(&i); + * const upb_value val = upb_strtable_iter_value(&i); + * // ... + * } + */ + +typedef struct { + const upb_strtable *t; + size_t index; +} upb_strtable_iter; + +void upb_strtable_begin(upb_strtable_iter *i, const upb_strtable *t); +void upb_strtable_next(upb_strtable_iter *i); +bool upb_strtable_done(const upb_strtable_iter *i); +const char *upb_strtable_iter_key(const upb_strtable_iter *i); +size_t upb_strtable_iter_keylength(const upb_strtable_iter *i); +upb_value upb_strtable_iter_value(const upb_strtable_iter *i); +void upb_strtable_iter_setdone(upb_strtable_iter *i); +bool upb_strtable_iter_isequal(const upb_strtable_iter *i1, + const upb_strtable_iter *i2); + + +/* upb_inttable_iter **********************************************************/ + +/* upb_inttable_iter i; + * upb_inttable_begin(&i, t); + * for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + * uintptr_t key = upb_inttable_iter_key(&i); + * upb_value val = upb_inttable_iter_value(&i); + * // ... + * } + */ + +typedef struct { + const upb_inttable *t; + size_t index; + bool array_part; +} upb_inttable_iter; + +void upb_inttable_begin(upb_inttable_iter *i, const upb_inttable *t); +void upb_inttable_next(upb_inttable_iter *i); +bool upb_inttable_done(const upb_inttable_iter *i); +uintptr_t upb_inttable_iter_key(const upb_inttable_iter *i); +upb_value upb_inttable_iter_value(const upb_inttable_iter *i); +void upb_inttable_iter_setdone(upb_inttable_iter *i); +bool upb_inttable_iter_isequal(const upb_inttable_iter *i1, + const upb_inttable_iter *i2); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* UPB_TABLE_H_ */ diff --git a/third_party/upb/upb/upb.c b/third_party/upb/upb/upb.c new file mode 100644 index 00000000000..266ea7d7f98 --- /dev/null +++ b/third_party/upb/upb/upb.c @@ -0,0 +1,261 @@ + +#include "upb/upb.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "upb/port_def.inc" + +/* Guarantee null-termination and provide ellipsis truncation. + * It may be tempting to "optimize" this by initializing these final + * four bytes up-front and then being careful never to overwrite them, + * this is safer and simpler. */ +static void nullz(upb_status *status) { + const char *ellipsis = "..."; + size_t len = strlen(ellipsis); + UPB_ASSERT(sizeof(status->msg) > len); + memcpy(status->msg + sizeof(status->msg) - len, ellipsis, len); +} + +/* upb_status *****************************************************************/ + +void upb_status_clear(upb_status *status) { + if (!status) return; + status->ok = true; + status->msg[0] = '\0'; +} + +bool upb_ok(const upb_status *status) { return status->ok; } + +const char *upb_status_errmsg(const upb_status *status) { return status->msg; } + +void upb_status_seterrmsg(upb_status *status, const char *msg) { + if (!status) return; + status->ok = false; + strncpy(status->msg, msg, sizeof(status->msg)); + nullz(status); +} + +void upb_status_seterrf(upb_status *status, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + upb_status_vseterrf(status, fmt, args); + va_end(args); +} + +void upb_status_vseterrf(upb_status *status, const char *fmt, va_list args) { + if (!status) return; + status->ok = false; + _upb_vsnprintf(status->msg, sizeof(status->msg), fmt, args); + nullz(status); +} + +/* upb_alloc ******************************************************************/ + +static void *upb_global_allocfunc(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size) { + UPB_UNUSED(alloc); + UPB_UNUSED(oldsize); + if (size == 0) { + free(ptr); + return NULL; + } else { + return realloc(ptr, size); + } +} + +upb_alloc upb_alloc_global = {&upb_global_allocfunc}; + +/* upb_arena ******************************************************************/ + +/* Be conservative and choose 16 in case anyone is using SSE. */ +static const size_t maxalign = 16; + +static size_t align_up_max(size_t size) { + return ((size + maxalign - 1) / maxalign) * maxalign; +} + +struct upb_arena { + /* We implement the allocator interface. + * This must be the first member of upb_arena! */ + upb_alloc alloc; + + /* Allocator to allocate arena blocks. We are responsible for freeing these + * when we are destroyed. */ + upb_alloc *block_alloc; + + size_t bytes_allocated; + size_t next_block_size; + size_t max_block_size; + + /* Linked list of blocks. Points to an arena_block, defined in env.c */ + void *block_head; + + /* Cleanup entries. Pointer to a cleanup_ent, defined in env.c */ + void *cleanup_head; +}; + +typedef struct mem_block { + struct mem_block *next; + size_t size; + size_t used; + bool owned; + /* Data follows. */ +} mem_block; + +typedef struct cleanup_ent { + struct cleanup_ent *next; + upb_cleanup_func *cleanup; + void *ud; +} cleanup_ent; + +static void upb_arena_addblock(upb_arena *a, void *ptr, size_t size, + bool owned) { + mem_block *block = ptr; + + block->next = a->block_head; + block->size = size; + block->used = align_up_max(sizeof(mem_block)); + block->owned = owned; + + a->block_head = block; + + /* TODO(haberman): ASAN poison. */ +} + +static mem_block *upb_arena_allocblock(upb_arena *a, size_t size) { + size_t block_size = UPB_MAX(size, a->next_block_size) + sizeof(mem_block); + mem_block *block = upb_malloc(a->block_alloc, block_size); + + if (!block) { + return NULL; + } + + upb_arena_addblock(a, block, block_size, true); + a->next_block_size = UPB_MIN(block_size * 2, a->max_block_size); + + return block; +} + +static void *upb_arena_doalloc(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size) { + upb_arena *a = (upb_arena*)alloc; /* upb_alloc is initial member. */ + mem_block *block = a->block_head; + void *ret; + + if (size == 0) { + return NULL; /* We are an arena, don't need individual frees. */ + } + + size = align_up_max(size); + + /* TODO(haberman): special-case if this is a realloc of the last alloc? */ + + if (!block || block->size - block->used < size) { + /* Slow path: have to allocate a new block. */ + block = upb_arena_allocblock(a, size); + + if (!block) { + return NULL; /* Out of memory. */ + } + } + + ret = (char*)block + block->used; + block->used += size; + + if (oldsize > 0) { + memcpy(ret, ptr, oldsize); /* Preserve existing data. */ + } + + /* TODO(haberman): ASAN unpoison. */ + + a->bytes_allocated += size; + return ret; +} + +/* Public Arena API ***********************************************************/ + +#define upb_alignof(type) offsetof (struct { char c; type member; }, member) + +upb_arena *upb_arena_init(void *mem, size_t n, upb_alloc *alloc) { + const size_t first_block_overhead = sizeof(upb_arena) + sizeof(mem_block); + upb_arena *a; + bool owned = false; + + /* Round block size down to alignof(*a) since we will allocate the arena + * itself at the end. */ + n &= ~(upb_alignof(upb_arena) - 1); + + if (n < first_block_overhead) { + /* We need to malloc the initial block. */ + n = first_block_overhead + 256; + owned = true; + if (!alloc || !(mem = upb_malloc(alloc, n))) { + return NULL; + } + } + + a = (void*)((char*)mem + n - sizeof(*a)); + n -= sizeof(*a); + + a->alloc.func = &upb_arena_doalloc; + a->block_alloc = &upb_alloc_global; + a->bytes_allocated = 0; + a->next_block_size = 256; + a->max_block_size = 16384; + a->cleanup_head = NULL; + a->block_head = NULL; + a->block_alloc = alloc; + + upb_arena_addblock(a, mem, n, owned); + + return a; +} + +#undef upb_alignof + +void upb_arena_free(upb_arena *a) { + cleanup_ent *ent = a->cleanup_head; + mem_block *block = a->block_head; + + while (ent) { + ent->cleanup(ent->ud); + ent = ent->next; + } + + /* Must do this after running cleanup functions, because this will delete + * the memory we store our cleanup entries in! */ + while (block) { + /* Load first since we are deleting block. */ + mem_block *next = block->next; + + if (block->owned) { + upb_free(a->block_alloc, block); + } + + block = next; + } +} + +bool upb_arena_addcleanup(upb_arena *a, void *ud, upb_cleanup_func *func) { + cleanup_ent *ent = upb_malloc(&a->alloc, sizeof(cleanup_ent)); + if (!ent) { + return false; /* Out of memory. */ + } + + ent->cleanup = func; + ent->ud = ud; + ent->next = a->cleanup_head; + a->cleanup_head = ent; + + return true; +} + +size_t upb_arena_bytesallocated(const upb_arena *a) { + return a->bytes_allocated; +} diff --git a/third_party/upb/upb/upb.h b/third_party/upb/upb/upb.h new file mode 100644 index 00000000000..79c19d281e9 --- /dev/null +++ b/third_party/upb/upb/upb.h @@ -0,0 +1,364 @@ +/* +** This file contains shared definitions that are widely used across upb. +** +** This is a mixed C/C++ interface that offers a full API to both languages. +** See the top-level README for more information. +*/ + +#ifndef UPB_H_ +#define UPB_H_ + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +#include +namespace upb { +class Arena; +class Status; +template class InlinedArena; +} +#endif + +#include "upb/port_def.inc" + +/* upb_status *****************************************************************/ + +/* upb_status represents a success or failure status and error message. + * It owns no resources and allocates no memory, so it should work + * even in OOM situations. */ + +/* The maximum length of an error message before it will get truncated. */ +#define UPB_STATUS_MAX_MESSAGE 127 + +typedef struct { + bool ok; + char msg[UPB_STATUS_MAX_MESSAGE]; /* Error message; NULL-terminated. */ +} upb_status; + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_status_errmsg(const upb_status *status); +bool upb_ok(const upb_status *status); + +/* Any of the functions that write to a status object allow status to be NULL, + * to support use cases where the function's caller does not care about the + * status message. */ +void upb_status_clear(upb_status *status); +void upb_status_seterrmsg(upb_status *status, const char *msg); +void upb_status_seterrf(upb_status *status, const char *fmt, ...); +void upb_status_vseterrf(upb_status *status, const char *fmt, va_list args); + +UPB_INLINE void upb_status_setoom(upb_status *status) { + upb_status_seterrmsg(status, "out of memory"); +} + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::Status { + public: + Status() { upb_status_clear(&status_); } + + upb_status* ptr() { return &status_; } + + /* Returns true if there is no error. */ + bool ok() const { return upb_ok(&status_); } + + /* Guaranteed to be NULL-terminated. */ + const char *error_message() const { return upb_status_errmsg(&status_); } + + /* The error message will be truncated if it is longer than + * UPB_STATUS_MAX_MESSAGE-4. */ + void SetErrorMessage(const char *msg) { upb_status_seterrmsg(&status_, msg); } + void SetFormattedErrorMessage(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + upb_status_vseterrf(&status_, fmt, args); + va_end(args); + } + + /* Resets the status to a successful state with no message. */ + void Clear() { upb_status_clear(&status_); } + + private: + upb_status status_; +}; + +#endif /* __cplusplus */ + +/** upb_strview ************************************************************/ + +typedef struct { + const char *data; + size_t size; +} upb_strview; + +UPB_INLINE upb_strview upb_strview_make(const char *data, size_t size) { + upb_strview ret; + ret.data = data; + ret.size = size; + return ret; +} + +UPB_INLINE upb_strview upb_strview_makez(const char *data) { + return upb_strview_make(data, strlen(data)); +} + +UPB_INLINE bool upb_strview_eql(upb_strview a, upb_strview b) { + return a.size == b.size && memcmp(a.data, b.data, a.size) == 0; +} + +#define UPB_STRVIEW_INIT(ptr, len) {ptr, len} + +#define UPB_STRVIEW_FORMAT "%.*s" +#define UPB_STRVIEW_ARGS(view) (int)(view).size, (view).data + +/** upb_alloc *****************************************************************/ + +/* A upb_alloc is a possibly-stateful allocator object. + * + * It could either be an arena allocator (which doesn't require individual + * free() calls) or a regular malloc() (which does). The client must therefore + * free memory unless it knows that the allocator is an arena allocator. */ + +struct upb_alloc; +typedef struct upb_alloc upb_alloc; + +/* A malloc()/free() function. + * If "size" is 0 then the function acts like free(), otherwise it acts like + * realloc(). Only "oldsize" bytes from a previous allocation are preserved. */ +typedef void *upb_alloc_func(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size); + +struct upb_alloc { + upb_alloc_func *func; +}; + +UPB_INLINE void *upb_malloc(upb_alloc *alloc, size_t size) { + UPB_ASSERT(alloc); + return alloc->func(alloc, NULL, 0, size); +} + +UPB_INLINE void *upb_realloc(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size) { + UPB_ASSERT(alloc); + return alloc->func(alloc, ptr, oldsize, size); +} + +UPB_INLINE void upb_free(upb_alloc *alloc, void *ptr) { + assert(alloc); + alloc->func(alloc, ptr, 0, 0); +} + +/* The global allocator used by upb. Uses the standard malloc()/free(). */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern upb_alloc upb_alloc_global; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +/* Functions that hard-code the global malloc. + * + * We still get benefit because we can put custom logic into our global + * allocator, like injecting out-of-memory faults in debug/testing builds. */ + +UPB_INLINE void *upb_gmalloc(size_t size) { + return upb_malloc(&upb_alloc_global, size); +} + +UPB_INLINE void *upb_grealloc(void *ptr, size_t oldsize, size_t size) { + return upb_realloc(&upb_alloc_global, ptr, oldsize, size); +} + +UPB_INLINE void upb_gfree(void *ptr) { + upb_free(&upb_alloc_global, ptr); +} + +/* upb_arena ******************************************************************/ + +/* upb_arena is a specific allocator implementation that uses arena allocation. + * The user provides an allocator that will be used to allocate the underlying + * arena blocks. Arenas by nature do not require the individual allocations + * to be freed. However the Arena does allow users to register cleanup + * functions that will run when the arena is destroyed. + * + * A upb_arena is *not* thread-safe. + * + * You could write a thread-safe arena allocator that satisfies the + * upb_alloc interface, but it would not be as efficient for the + * single-threaded case. */ + +typedef void upb_cleanup_func(void *ud); + +struct upb_arena; +typedef struct upb_arena upb_arena; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Creates an arena from the given initial block (if any -- n may be 0). + * Additional blocks will be allocated from |alloc|. If |alloc| is NULL, this + * is a fixed-size arena and cannot grow. */ +upb_arena *upb_arena_init(void *mem, size_t n, upb_alloc *alloc); +void upb_arena_free(upb_arena *a); +bool upb_arena_addcleanup(upb_arena *a, void *ud, upb_cleanup_func *func); +size_t upb_arena_bytesallocated(const upb_arena *a); + +UPB_INLINE upb_alloc *upb_arena_alloc(upb_arena *a) { return (upb_alloc*)a; } + +/* Convenience wrappers around upb_alloc functions. */ + +UPB_INLINE void *upb_arena_malloc(upb_arena *a, size_t size) { + return upb_malloc(upb_arena_alloc(a), size); +} + +UPB_INLINE void *upb_arena_realloc(upb_arena *a, void *ptr, size_t oldsize, + size_t size) { + return upb_realloc(upb_arena_alloc(a), ptr, oldsize, size); +} + +UPB_INLINE upb_arena *upb_arena_new(void) { + return upb_arena_init(NULL, 0, &upb_alloc_global); +} + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::Arena { + public: + /* A simple arena with no initial memory block and the default allocator. */ + Arena() : ptr_(upb_arena_new(), upb_arena_free) {} + + upb_arena* ptr() { return ptr_.get(); } + + /* Allows this arena to be used as a generic allocator. + * + * The arena does not need free() calls so when using Arena as an allocator + * it is safe to skip them. However they are no-ops so there is no harm in + * calling free() either. */ + upb_alloc *allocator() { return upb_arena_alloc(ptr_.get()); } + + /* Add a cleanup function to run when the arena is destroyed. + * Returns false on out-of-memory. */ + bool AddCleanup(void *ud, upb_cleanup_func* func) { + return upb_arena_addcleanup(ptr_.get(), ud, func); + } + + /* Total number of bytes that have been allocated. It is undefined what + * Realloc() does to &arena_ counter. */ + size_t BytesAllocated() const { return upb_arena_bytesallocated(ptr_.get()); } + + private: + std::unique_ptr ptr_; +}; + +#endif + +/* upb::InlinedArena **********************************************************/ + +/* upb::InlinedArena seeds the arenas with a predefined amount of memory. No + * heap memory will be allocated until the initial block is exceeded. + * + * These types only exist in C++ */ + +#ifdef __cplusplus + +template class upb::InlinedArena : public upb::Arena { + public: + InlinedArena() : ptr_(upb_arena_new(&initial_block_, N, &upb_alloc_global)) {} + + upb_arena* ptr() { return ptr_.get(); } + + private: + InlinedArena(const InlinedArena*) = delete; + InlinedArena& operator=(const InlinedArena*) = delete; + + std::unique_ptr ptr_; + char initial_block_[N]; +}; + +#endif /* __cplusplus */ + +/* Constants ******************************************************************/ + +/* Generic function type. */ +typedef void upb_func(void); + +/* A list of types as they are encoded on-the-wire. */ +typedef enum { + UPB_WIRE_TYPE_VARINT = 0, + UPB_WIRE_TYPE_64BIT = 1, + UPB_WIRE_TYPE_DELIMITED = 2, + UPB_WIRE_TYPE_START_GROUP = 3, + UPB_WIRE_TYPE_END_GROUP = 4, + UPB_WIRE_TYPE_32BIT = 5 +} upb_wiretype_t; + +/* The types a field can have. Note that this list is not identical to the + * types defined in descriptor.proto, which gives INT32 and SINT32 separate + * types (we distinguish the two with the "integer encoding" enum below). */ +typedef enum { + /* Types stored in 1 byte. */ + UPB_TYPE_BOOL = 1, + /* Types stored in 4 bytes. */ + UPB_TYPE_FLOAT = 2, + UPB_TYPE_INT32 = 3, + UPB_TYPE_UINT32 = 4, + UPB_TYPE_ENUM = 5, /* Enum values are int32. */ + /* Types stored as pointers (probably 4 or 8 bytes). */ + UPB_TYPE_STRING = 6, + UPB_TYPE_BYTES = 7, + UPB_TYPE_MESSAGE = 8, + /* Types stored as 8 bytes. */ + UPB_TYPE_DOUBLE = 9, + UPB_TYPE_INT64 = 10, + UPB_TYPE_UINT64 = 11 +} upb_fieldtype_t; + +/* The repeated-ness of each field; this matches descriptor.proto. */ +typedef enum { + UPB_LABEL_OPTIONAL = 1, + UPB_LABEL_REQUIRED = 2, + UPB_LABEL_REPEATED = 3 +} upb_label_t; + +/* Descriptor types, as defined in descriptor.proto. */ +typedef enum { + UPB_DESCRIPTOR_TYPE_DOUBLE = 1, + UPB_DESCRIPTOR_TYPE_FLOAT = 2, + UPB_DESCRIPTOR_TYPE_INT64 = 3, + UPB_DESCRIPTOR_TYPE_UINT64 = 4, + UPB_DESCRIPTOR_TYPE_INT32 = 5, + UPB_DESCRIPTOR_TYPE_FIXED64 = 6, + UPB_DESCRIPTOR_TYPE_FIXED32 = 7, + UPB_DESCRIPTOR_TYPE_BOOL = 8, + UPB_DESCRIPTOR_TYPE_STRING = 9, + UPB_DESCRIPTOR_TYPE_GROUP = 10, + UPB_DESCRIPTOR_TYPE_MESSAGE = 11, + UPB_DESCRIPTOR_TYPE_BYTES = 12, + UPB_DESCRIPTOR_TYPE_UINT32 = 13, + UPB_DESCRIPTOR_TYPE_ENUM = 14, + UPB_DESCRIPTOR_TYPE_SFIXED32 = 15, + UPB_DESCRIPTOR_TYPE_SFIXED64 = 16, + UPB_DESCRIPTOR_TYPE_SINT32 = 17, + UPB_DESCRIPTOR_TYPE_SINT64 = 18 +} upb_descriptortype_t; + +extern const uint8_t upb_desctype_to_fieldtype[]; + +#include "upb/port_undef.inc" + +#endif /* UPB_H_ */ diff --git a/third_party/upb/upbc/generator.cc b/third_party/upb/upbc/generator.cc new file mode 100644 index 00000000000..7096278180b --- /dev/null +++ b/third_party/upb/upbc/generator.cc @@ -0,0 +1,826 @@ + +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "google/protobuf/io/zero_copy_stream.h" + +#include "upbc/generator.h" +#include "upbc/message_layout.h" + +namespace protoc = ::google::protobuf::compiler; +namespace protobuf = ::google::protobuf; + +static std::string StripExtension(absl::string_view fname) { + size_t lastdot = fname.find_last_of("."); + if (lastdot == std::string::npos) { + return std::string(fname); + } + return std::string(fname.substr(0, lastdot)); +} + +static std::string HeaderFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upb.h"; +} + +static std::string SourceFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upb.c"; +} + +static std::string DefHeaderFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upbdefs.h"; +} + +static std::string DefSourceFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upbdefs.c"; +} + +class Output { + public: + Output(protobuf::io::ZeroCopyOutputStream* stream) : stream_(stream) {} + ~Output() { stream_->BackUp(size_); } + + template + void operator()(absl::string_view format, const Arg&... arg) { + Write(absl::Substitute(format, arg...)); + } + + private: + void Write(absl::string_view data) { + while (!data.empty()) { + RefreshOutput(); + size_t to_write = std::min(data.size(), size_); + memcpy(ptr_, data.data(), to_write); + data.remove_prefix(to_write); + ptr_ += to_write; + size_ -= to_write; + } + } + + void RefreshOutput() { + while (size_ == 0) { + void *ptr; + int size; + if (!stream_->Next(&ptr, &size)) { + fprintf(stderr, "upbc: Failed to write to to output\n"); + abort(); + } + ptr_ = static_cast(ptr); + size_ = size; + } + } + + protobuf::io::ZeroCopyOutputStream* stream_; + char *ptr_ = nullptr; + size_t size_ = 0; +}; + +namespace upbc { + +class Generator : public protoc::CodeGenerator { + ~Generator() override {} + bool Generate(const protobuf::FileDescriptor* file, + const std::string& parameter, protoc::GeneratorContext* context, + std::string* error) const override; + +}; + +void AddMessages(const protobuf::Descriptor* message, + std::vector* messages) { + messages->push_back(message); + for (int i = 0; i < message->nested_type_count(); i++) { + AddMessages(message->nested_type(i), messages); + } +} + +void AddEnums(const protobuf::Descriptor* message, + std::vector* enums) { + for (int i = 0; i < message->enum_type_count(); i++) { + enums->push_back(message->enum_type(i)); + } + for (int i = 0; i < message->nested_type_count(); i++) { + AddEnums(message->nested_type(i), enums); + } +} + +template +void SortDefs(std::vector* defs) { + std::sort(defs->begin(), defs->end(), + [](T a, T b) { return a->full_name() < b->full_name(); }); +} + +std::vector SortedMessages( + const protobuf::FileDescriptor* file) { + std::vector messages; + for (int i = 0; i < file->message_type_count(); i++) { + AddMessages(file->message_type(i), &messages); + } + return messages; +} + +std::vector SortedEnums( + const protobuf::FileDescriptor* file) { + std::vector enums; + for (int i = 0; i < file->enum_type_count(); i++) { + enums.push_back(file->enum_type(i)); + } + for (int i = 0; i < file->message_type_count(); i++) { + AddEnums(file->message_type(i), &enums); + } + SortDefs(&enums); + return enums; +} + +std::vector FieldNumberOrder( + const protobuf::Descriptor* message) { + std::vector messages; + for (int i = 0; i < message->field_count(); i++) { + messages.push_back(message->field(i)); + } + std::sort(messages.begin(), messages.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return a->number() < b->number(); + }); + return messages; +} + +std::vector SortedSubmessages( + const protobuf::Descriptor* message) { + std::vector ret; + for (int i = 0; i < message->field_count(); i++) { + if (message->field(i)->cpp_type() == + protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + ret.push_back(message->field(i)); + } + } + std::sort(ret.begin(), ret.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return a->message_type()->full_name() < + b->message_type()->full_name(); + }); + return ret; +} + +std::string ToCIdent(absl::string_view str) { + return absl::StrReplaceAll(str, {{".", "_"}, {"/", "_"}}); +} + +std::string DefInitSymbol(const protobuf::FileDescriptor *file) { + return ToCIdent(file->name()) + "_upbdefinit"; +} + +std::string ToPreproc(absl::string_view str) { + return absl::AsciiStrToUpper(ToCIdent(str)); +} + +std::string EnumValueSymbol(const protobuf::EnumValueDescriptor* value) { + return ToCIdent(value->full_name()); +} + +std::string GetSizeInit(const MessageLayout::Size& size) { + return absl::Substitute("UPB_SIZE($0, $1)", size.size32, size.size64); +} + +std::string MessageName(const protobuf::Descriptor* descriptor) { + return ToCIdent(descriptor->full_name()); +} + +std::string MessageInit(const protobuf::Descriptor* descriptor) { + return MessageName(descriptor) + "_msginit"; +} + +std::string CTypeInternal(const protobuf::FieldDescriptor* field, + bool is_const) { + std::string maybe_const = is_const ? "const " : ""; + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + std::string maybe_struct = + field->file() != field->message_type()->file() ? "struct " : ""; + return maybe_const + maybe_struct + MessageName(field->message_type()) + + "*"; + } + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return "bool"; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return "float"; + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return "int32_t"; + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return "uint32_t"; + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return "double"; + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return "int64_t"; + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return "uint64_t"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return "upb_strview"; + default: + fprintf(stderr, "Unexpected type"); + abort(); + } +} + +std::string UpbType(const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return "UPB_TYPE_MESSAGE"; + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return "UPB_TYPE_ENUM"; + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return "UPB_TYPE_BOOL"; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return "UPB_TYPE_FLOAT"; + case protobuf::FieldDescriptor::CPPTYPE_INT32: + return "UPB_TYPE_INT32"; + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return "UPB_TYPE_UINT32"; + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return "UPB_TYPE_DOUBLE"; + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return "UPB_TYPE_INT64"; + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return "UPB_TYPE_UINT64"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return "UPB_TYPE_STRING"; + default: + fprintf(stderr, "Unexpected type"); + abort(); + } +} + +std::string FieldDefault(const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return "NULL"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return absl::Substitute("upb_strview_make(\"$0\", strlen(\"$0\"))", + absl::CEscape(field->default_value_string())); + case protobuf::FieldDescriptor::CPPTYPE_INT32: + return absl::StrCat(field->default_value_int32()); + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return absl::StrCat(field->default_value_int64()); + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return absl::StrCat(field->default_value_uint32()); + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return absl::StrCat(field->default_value_uint64()); + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return absl::StrCat(field->default_value_float()); + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return absl::StrCat(field->default_value_double()); + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return field->default_value_bool() ? "true" : "false"; + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return EnumValueSymbol(field->default_value_enum()); + } + ABSL_ASSERT(false); + return "XXX"; +} + +std::string CType(const protobuf::FieldDescriptor* field) { + return CTypeInternal(field, false); +} + +std::string CTypeConst(const protobuf::FieldDescriptor* field) { + return CTypeInternal(field, true); +} + +void DumpEnumValues(const protobuf::EnumDescriptor* desc, Output& output) { + std::vector values; + for (int i = 0; i < desc->value_count(); i++) { + values.push_back(desc->value(i)); + } + std::sort(values.begin(), values.end(), + [](const protobuf::EnumValueDescriptor* a, + const protobuf::EnumValueDescriptor* b) { + return a->number() < b->number(); + }); + + for (size_t i = 0; i < values.size(); i++) { + auto value = values[i]; + output(" $0 = $1", EnumValueSymbol(value), value->number()); + if (i != values.size() - 1) { + output(","); + } + output("\n"); + } +} + +void EmitFileWarning(const protobuf::FileDescriptor* file, Output& output) { + output( + "/* This file was generated by upbc (the upb compiler) from the input\n" + " * file:\n" + " *\n" + " * $0\n" + " *\n" + " * Do not edit -- your changes will be discarded when the file is\n" + " * regenerated. */\n\n", + file->name()); +} + +void GenerateMessageInHeader(const protobuf::Descriptor* message, Output& output) { + MessageLayout layout(message); + + output("/* $0 */\n\n", message->full_name()); + std::string msgname = ToCIdent(message->full_name()); + output( + "UPB_INLINE $0 *$0_new(upb_arena *arena) {\n" + " return ($0 *)upb_msg_new(&$1, arena);\n" + "}\n" + "UPB_INLINE $0 *$0_parse(const char *buf, size_t size,\n" + " upb_arena *arena) {\n" + " $0 *ret = $0_new(arena);\n" + " return (ret && upb_decode(buf, size, ret, &$1, arena)) ? ret : NULL;\n" + "}\n" + "UPB_INLINE char *$0_serialize(const $0 *msg, upb_arena *arena, size_t " + "*len) {\n" + " return upb_encode(msg, &$1, arena, len);\n" + "}\n" + "\n", + MessageName(message), MessageInit(message)); + + for (int i = 0; i < message->oneof_decl_count(); i++) { + const protobuf::OneofDescriptor* oneof = message->oneof_decl(i); + std::string fullname = ToCIdent(oneof->full_name()); + output("typedef enum {\n"); + for (int j = 0; j < oneof->field_count(); j++) { + const protobuf::FieldDescriptor* field = oneof->field(j); + output(" $0_$1 = $2,\n", fullname, field->name(), field->number()); + } + output( + " $0_NOT_SET = 0\n" + "} $0_oneofcases;\n", + fullname); + output( + "UPB_INLINE $0_oneofcases $1_$2_case(const $1* msg) { " + "return ($0_oneofcases)UPB_FIELD_AT(msg, int32_t, $3); }\n" + "\n", + fullname, msgname, oneof->name(), + GetSizeInit(layout.GetOneofCaseOffset(oneof))); + } + + for (auto field : FieldNumberOrder(message)) { + + if (layout.HasHasbit(field)) { + output( + "UPB_INLINE bool $0_has_$1(const $0 *msg) { " + "return _upb_has_field(msg, $2); }\n", + msgname, field->name(), layout.GetHasbitIndex(field)); + } else if (field->containing_oneof()) { + output( + "UPB_INLINE bool $0_has_$1(const $0 *msg) { " + "return _upb_has_oneof_field(msg, $2, $3); }\n", + msgname, field->name(), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number()); + } + + if (field->is_repeated()) { + output( + "UPB_INLINE $0 const* $1_$2(const $1 *msg, size_t *len) { " + "return ($0 const*)_upb_array_accessor(msg, $3, len); }\n", + CTypeConst(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field))); + } else if (field->containing_oneof()) { + output( + "UPB_INLINE $0 $1_$2(const $1 *msg) { " + "return UPB_READ_ONEOF(msg, $0, $3, $4, $5, $6); }\n", + CTypeConst(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number(), FieldDefault(field)); + } else { + output( + "UPB_INLINE $0 $1_$2(const $1 *msg) { " + "return UPB_FIELD_AT(msg, $0, $3); }\n", + CTypeConst(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field))); + } + } + + output("\n"); + + for (auto field : FieldNumberOrder(message)) { + if (field->is_repeated()) { + output( + "UPB_INLINE $0* $1_mutable_$2($1 *msg, size_t *len) {\n" + " return ($0*)_upb_array_mutable_accessor(msg, $3, len);\n" + "}\n", + CType(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field))); + output( + "UPB_INLINE $0* $1_resize_$2($1 *msg, size_t len, " + "upb_arena *arena) {\n" + " return ($0*)_upb_array_resize_accessor(msg, $3, len, $4, $5, " + "arena);\n" + "}\n", + CType(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(MessageLayout::SizeOfUnwrapped(field).size), + UpbType(field)); + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + output( + "UPB_INLINE struct $0* $1_add_$2($1 *msg, upb_arena *arena) {\n" + " struct $0* sub = (struct $0*)upb_msg_new(&$3, arena);\n" + " bool ok = _upb_array_append_accessor(\n" + " msg, $4, $5, $6, &sub, arena);\n" + " if (!ok) return NULL;\n" + " return sub;\n" + "}\n", + MessageName(field->message_type()), msgname, field->name(), + MessageInit(field->message_type()), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(MessageLayout::SizeOfUnwrapped(field).size), + UpbType(field)); + } else { + output( + "UPB_INLINE bool $1_add_$2($1 *msg, $0 val, upb_arena *arena) {\n" + " return _upb_array_append_accessor(\n" + " msg, $3, $4, $5, &val, arena);\n" + "}\n", + CType(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(MessageLayout::SizeOfUnwrapped(field).size), + UpbType(field)); + } + } else { + output("UPB_INLINE void $0_set_$1($0 *msg, $2 value) {\n", msgname, + field->name(), CType(field)); + if (field->containing_oneof()) { + output( + " UPB_WRITE_ONEOF(msg, $0, $1, value, $2, $3);\n" + "}\n", + CType(field), GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number()); + } else { + if (MessageLayout::HasHasbit(field)) { + output(" _upb_sethas(msg, $0);\n", layout.GetHasbitIndex(field)); + } + output( + " UPB_FIELD_AT(msg, $0, $1) = value;\n" + "}\n", + CType(field), GetSizeInit(layout.GetFieldOffset(field))); + } + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + output( + "UPB_INLINE struct $0* $1_mutable_$2($1 *msg, upb_arena *arena) {\n" + " struct $0* sub = (struct $0*)$1_$2(msg);\n" + " if (sub == NULL) {\n" + " sub = (struct $0*)upb_msg_new(&$3, arena);\n" + " if (!sub) return NULL;\n" + " $1_set_$2(msg, sub);\n" + " }\n" + " return sub;\n" + "}\n", + MessageName(field->message_type()), msgname, field->name(), + MessageInit(field->message_type())); + } + } + } + + output("\n"); +} + +void WriteHeader(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + output( + "#ifndef $0_UPB_H_\n" + "#define $0_UPB_H_\n\n" + "#include \"upb/generated_util.h\"\n" + "#include \"upb/msg.h\"\n" + "#include \"upb/decode.h\"\n" + "#include \"upb/encode.h\"\n\n", + ToPreproc(file->name())); + + for (int i = 0; i < file->public_dependency_count(); i++) { + const auto& name = file->public_dependency(i)->name(); + if (i == 0) { + output("/* Public Imports. */\n"); + } + output("#include \"$0\"\n", HeaderFilename(name)); + if (i == file->public_dependency_count() - 1) { + output("\n"); + } + } + + output( + "#include \"upb/port_def.inc\"\n" + "\n" + "#ifdef __cplusplus\n" + "extern \"C\" {\n" + "#endif\n" + "\n"); + + std::vector this_file_messages = + SortedMessages(file); + + // Forward-declare types defined in this file. + for (auto message : this_file_messages) { + output("struct $0;\n", ToCIdent(message->full_name())); + } + for (auto message : this_file_messages) { + output("typedef struct $0 $0;\n", ToCIdent(message->full_name())); + } + for (auto message : this_file_messages) { + output("extern const upb_msglayout $0;\n", MessageInit(message)); + } + + // Forward-declare types not in this file, but used as submessages. + // Order by full name for consistent ordering. + std::map forward_messages; + + for (auto message : SortedMessages(file)) { + for (int i = 0; i < message->field_count(); i++) { + const protobuf::FieldDescriptor* field = message->field(i); + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->file() != field->message_type()->file()) { + forward_messages[field->message_type()->full_name()] = + field->message_type(); + } + } + } + for (const auto& pair : forward_messages) { + output("struct $0;\n", MessageName(pair.second)); + } + for (const auto& pair : forward_messages) { + output("extern const upb_msglayout $0;\n", MessageInit(pair.second)); + } + + if (!this_file_messages.empty()) { + output("\n"); + } + + std::vector this_file_enums = + SortedEnums(file); + + for (auto enumdesc : this_file_enums) { + output("typedef enum {\n"); + DumpEnumValues(enumdesc, output); + output("} $0;\n\n", ToCIdent(enumdesc->full_name())); + } + + output("\n"); + + for (auto message : this_file_messages) { + GenerateMessageInHeader(message, output); + } + + output( + "#ifdef __cplusplus\n" + "} /* extern \"C\" */\n" + "#endif\n" + "\n" + "#include \"upb/port_undef.inc\"\n" + "\n" + "#endif /* $0_UPB_H_ */\n", + ToPreproc(file->name())); +} + +void WriteSource(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output( + "#include \n" + "#include \"upb/msg.h\"\n" + "#include \"$0\"\n", + HeaderFilename(file->name())); + + for (int i = 0; i < file->dependency_count(); i++) { + output("#include \"$0\"\n", HeaderFilename(file->dependency(i)->name())); + } + + output( + "\n" + "#include \"upb/port_def.inc\"\n" + "\n"); + + + for (auto message : SortedMessages(file)) { + std::string msgname = ToCIdent(message->full_name()); + std::string fields_array_ref = "NULL"; + std::string submsgs_array_ref = "NULL"; + std::string oneofs_array_ref = "NULL"; + absl::flat_hash_map submsg_indexes; + MessageLayout layout(message); + std::vector sorted_submsgs = + SortedSubmessages(message); + + if (!sorted_submsgs.empty()) { + // TODO(haberman): could save a little bit of space by only generating a + // "submsgs" array for every strongly-connected component. + std::string submsgs_array_name = msgname + "_submsgs"; + submsgs_array_ref = "&" + submsgs_array_name + "[0]"; + output("static const upb_msglayout *const $0[$1] = {\n", + submsgs_array_name, sorted_submsgs.size()); + + int i = 0; + for (auto submsg : sorted_submsgs) { + if (submsg_indexes.find(submsg->message_type()) != + submsg_indexes.end()) { + continue; + } + output(" &$0,\n", MessageInit(submsg->message_type())); + submsg_indexes[submsg->message_type()] = i++; + } + + output("};\n\n"); + } + + std::vector field_number_order = + FieldNumberOrder(message); + if (!field_number_order.empty()) { + std::string fields_array_name = msgname + "__fields"; + fields_array_ref = "&" + fields_array_name + "[0]"; + output("static const upb_msglayout_field $0[$1] = {\n", + fields_array_name, field_number_order.size()); + for (auto field : field_number_order) { + int submsg_index = 0; + std::string presence = "0"; + + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + submsg_index = submsg_indexes[field->message_type()]; + } + + if (MessageLayout::HasHasbit(field)) { + presence = absl::StrCat(layout.GetHasbitIndex(field)); + } else if (field->containing_oneof()) { + MessageLayout::Size case_offset = + layout.GetOneofCaseOffset(field->containing_oneof()); + + // Our encoding that distinguishes oneofs from presence-having fields. + case_offset.size32 = -case_offset.size32 - 1; + case_offset.size64 = -case_offset.size64 - 1; + presence = GetSizeInit(case_offset); + } + + output(" {$0, $1, $2, $3, $4, $5},\n", + field->number(), + GetSizeInit(layout.GetFieldOffset(field)), + presence, + submsg_index, + field->type(), + field->label()); + } + output("};\n\n"); + } + + output("const upb_msglayout $0 = {\n", MessageInit(message)); + output(" $0,\n", submsgs_array_ref); + output(" $0,\n", fields_array_ref); + output(" $0, $1, $2,\n", GetSizeInit(layout.message_size()), + field_number_order.size(), + "false" // TODO: extendable + ); + + output("};\n\n"); + } + + output("#include \"upb/port_undef.inc\"\n"); + output("\n"); +} + +void GenerateMessageDefAccessor(const protobuf::Descriptor* d, Output& output) { + output("UPB_INLINE const upb_msgdef *$0_getmsgdef(upb_symtab *s) {\n", + ToCIdent(d->full_name())); + output(" _upb_symtab_loaddefinit(s, &$0);\n", DefInitSymbol(d->file())); + output(" return upb_symtab_lookupmsg(s, \"$0\");\n", d->full_name()); + output("}\n"); + output("\n"); + + for (int i = 0; i < d->nested_type_count(); i++) { + GenerateMessageDefAccessor(d->nested_type(i), output); + } +} + +void WriteDefHeader(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output( + "#ifndef $0_UPBDEFS_H_\n" + "#define $0_UPBDEFS_H_\n\n" + "#include \"upb/def.h\"\n" + "#include \"upb/port_def.inc\"\n" + "#ifdef __cplusplus\n" + "extern \"C\" {\n" + "#endif\n\n", + ToPreproc(file->name())); + + output("#include \"upb/def.h\"\n"); + output("\n"); + output("#include \"upb/port_def.inc\"\n"); + output("\n"); + + output("extern upb_def_init $0;\n", DefInitSymbol(file)); + output("\n"); + + for (int i = 0; i < file->message_type_count(); i++) { + GenerateMessageDefAccessor(file->message_type(i), output); + } + + output( + "#ifdef __cplusplus\n" + "} /* extern \"C\" */\n" + "#endif\n" + "\n" + "#include \"upb/port_undef.inc\"\n" + "\n" + "#endif /* $0_UPBDEFS_H_ */\n", + ToPreproc(file->name())); +} + +// Escape C++ trigraphs by escaping question marks to \? +std::string EscapeTrigraphs(absl::string_view to_escape) { + return absl::StrReplaceAll(to_escape, {{"?", "\\?"}}); +} + +void WriteDefSource(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output("#include \"upb/def.h\"\n"); + output("\n"); + + for (int i = 0; i < file->dependency_count(); i++) { + output("extern upb_def_init $0;\n", DefInitSymbol(file->dependency(i))); + } + + protobuf::FileDescriptorProto file_proto; + file->CopyTo(&file_proto); + std::string file_data; + file_proto.SerializeToString(&file_data); + + output("static const char descriptor[$0] =\n", file_data.size()); + + { + if (file_data.size() > 65535) { + // Workaround for MSVC: "Error C1091: compiler limit: string exceeds + // 65535 bytes in length". Declare a static array of chars rather than + // use a string literal. Only write 25 bytes per line. + static const size_t kBytesPerLine = 25; + output("{ "); + for (size_t i = 0; i < file_data.size();) { + for (size_t j = 0; j < kBytesPerLine && i < file_data.size(); ++i, ++j) { + output("'$0', ", absl::CEscape(file_data.substr(i, 1))); + } + output("\n"); + } + output("'\\0' }"); // null-terminate + } else { + // Only write 40 bytes per line. + static const size_t kBytesPerLine = 40; + for (size_t i = 0; i < file_data.size(); i += kBytesPerLine) { + output( + "\"$0\"\n", + EscapeTrigraphs(absl::CEscape(file_data.substr(i, kBytesPerLine)))); + } + } + output(";\n"); + } + + output("static upb_def_init *deps[$0] = {\n", file->dependency_count() + 1); + for (int i = 0; i < file->dependency_count(); i++) { + output(" &$0,\n", DefInitSymbol(file->dependency(i))); + } + output(" NULL\n"); + output("};\n"); + + output("upb_def_init $0 = {\n", DefInitSymbol(file)); + output(" deps,\n"); + output(" \"$0\",\n", file->name()); + output(" UPB_STRVIEW_INIT(descriptor, $0)\n", file_data.size()); + output("};\n"); +} + +bool Generator::Generate(const protobuf::FileDescriptor* file, + const std::string& parameter, + protoc::GeneratorContext* context, + std::string* error) const { + Output h_output(context->Open(HeaderFilename(file->name()))); + WriteHeader(file, h_output); + + Output c_output(context->Open(SourceFilename(file->name()))); + WriteSource(file, c_output); + + Output h_def_output(context->Open(DefHeaderFilename(file->name()))); + WriteDefHeader(file, h_def_output); + + Output c_def_output(context->Open(DefSourceFilename(file->name()))); + WriteDefSource(file, c_def_output); + + return true; +} + +std::unique_ptr GetGenerator() { + return std::unique_ptr( + new Generator()); +} + +} // namespace upbc diff --git a/third_party/upb/upbc/generator.h b/third_party/upb/upbc/generator.h new file mode 100644 index 00000000000..ed6cedc6c77 --- /dev/null +++ b/third_party/upb/upbc/generator.h @@ -0,0 +1,12 @@ + +#ifndef UPBC_GENERATOR_H_ +#define UPBC_GENERATOR_H_ + +#include +#include + +namespace upbc { +std::unique_ptr GetGenerator(); +} + +#endif // UPBC_GENERATOR_H_ diff --git a/third_party/upb/upbc/main.cc b/third_party/upb/upbc/main.cc new file mode 100644 index 00000000000..a9682a9c1a3 --- /dev/null +++ b/third_party/upb/upbc/main.cc @@ -0,0 +1,9 @@ + +#include + +#include "upbc/generator.h" + +int main(int argc, char** argv) { + return google::protobuf::compiler::PluginMain(argc, argv, + upbc::GetGenerator().get()); +} diff --git a/third_party/upb/upbc/message_layout.cc b/third_party/upb/upbc/message_layout.cc new file mode 100644 index 00000000000..f0a68725c20 --- /dev/null +++ b/third_party/upb/upbc/message_layout.cc @@ -0,0 +1,179 @@ + +#include "upbc/message_layout.h" + +namespace upbc { + +namespace protobuf = ::google::protobuf; + +static int64_t DivRoundUp(int64_t a, int64_t b) { + ABSL_ASSERT(a >= 0); + ABSL_ASSERT(b > 0); + return (a + b - 1) / b; +} + +MessageLayout::Size MessageLayout::Place( + MessageLayout::SizeAndAlign size_and_align) { + Size offset = size_; + offset.AlignUp(size_and_align.align); + size_ = offset; + size_.Add(size_and_align.size); + //maxalign_.MaxFrom(size_and_align.align); + maxalign_.MaxFrom(size_and_align.size); + return offset; +} + +bool MessageLayout::HasHasbit(const protobuf::FieldDescriptor* field) { + return field->file()->syntax() == protobuf::FileDescriptor::SYNTAX_PROTO2 && + field->label() != protobuf::FieldDescriptor::LABEL_REPEATED && + !field->containing_oneof(); +} + +MessageLayout::SizeAndAlign MessageLayout::SizeOf( + const protobuf::FieldDescriptor* field) { + if (field->is_repeated()) { + return {{4, 8}, {4, 8}}; // Pointer to array object. + } else { + return SizeOfUnwrapped(field); + } +} + +MessageLayout::SizeAndAlign MessageLayout::SizeOfUnwrapped( + const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return {{4, 8}, {4, 8}}; // Pointer to message. + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return {{8, 16}, {4, 8}}; // upb_stringview + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return {{1, 1}, {1, 1}}; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return {{4, 4}, {4, 4}}; + default: + return {{8, 8}, {8, 8}}; + } +} + +int64_t MessageLayout::FieldLayoutRank(const protobuf::FieldDescriptor* field) { + // Order: + // 1, 2, 3. primitive fields (8, 4, 1 byte) + // 4. string fields + // 5. submessage fields + // 6. repeated fields + // + // This has the following nice properties: + // + // 1. padding alignment is (nearly) minimized. + // 2. fields that might have defaults (1-4) are segregated + // from fields that are always zero-initialized (5-7). + // + // We skip oneof fields, because they are emitted in a separate pass. + int64_t rank; + if (field->containing_oneof()) { + fprintf(stderr, "shouldn't have oneofs here.\n"); + abort(); + } else if (field->label() == protobuf::FieldDescriptor::LABEL_REPEATED) { + rank = 6; + } else { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + rank = 5; + break; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + rank = 4; + break; + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + rank = 3; + break; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + rank = 2; + break; + default: + rank = 1; + break; + } + } + + // Break ties with field number. + return (rank << 29) | field->number(); +} + +void MessageLayout::ComputeLayout(const protobuf::Descriptor* descriptor) { + size_ = Size{0, 0}; + maxalign_ = Size{0, 0}; + PlaceNonOneofFields(descriptor); + PlaceOneofFields(descriptor); + + // Align overall size up to max size. + size_.AlignUp(maxalign_); +} + +void MessageLayout::PlaceNonOneofFields( + const protobuf::Descriptor* descriptor) { + std::vector field_order; + for (int i = 0; i < descriptor->field_count(); i++) { + const protobuf::FieldDescriptor* field = descriptor->field(i); + if (!field->containing_oneof()) { + field_order.push_back(descriptor->field(i)); + } + } + std::sort(field_order.begin(), field_order.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return FieldLayoutRank(a) < FieldLayoutRank(b); + }); + + // Place/count hasbits. + int hasbit_count = 0; + for (auto field : field_order) { + if (HasHasbit(field)) { + // We don't use hasbit 0, so that 0 can indicate "no presence" in the + // table. This wastes one hasbit, but we don't worry about it for now. + hasbit_indexes_[field] = ++hasbit_count; + } + } + + // Place hasbits at the beginning. + int64_t hasbit_bytes = DivRoundUp(hasbit_count, 8); + Place(SizeAndAlign{{hasbit_bytes, hasbit_bytes}, {1, 1}}); + + // Place non-oneof fields. + for (auto field : field_order) { + field_offsets_[field] = Place(SizeOf(field)); + } +} + +void MessageLayout::PlaceOneofFields(const protobuf::Descriptor* descriptor) { + std::vector oneof_order; + for (int i = 0; i < descriptor->oneof_decl_count(); i++) { + oneof_order.push_back(descriptor->oneof_decl(i)); + } + std::sort(oneof_order.begin(), oneof_order.end(), + [](const protobuf::OneofDescriptor* a, + const protobuf::OneofDescriptor* b) { + return a->full_name() < b->full_name(); + }); + + for (auto oneof : oneof_order) { + SizeAndAlign oneof_maxsize{{0, 0}, {0, 0}}; + // Calculate max size. + for (int i = 0; i < oneof->field_count(); i++) { + oneof_maxsize.MaxFrom(SizeOf(oneof->field(i))); + } + + // Place discriminator enum and data. + Size data = Place(oneof_maxsize); + Size discriminator = Place(SizeAndAlign{{4, 4}, {4, 4}}); + + oneof_case_offsets_[oneof] = discriminator; + + for (int i = 0; i < oneof->field_count(); i++) { + field_offsets_[oneof->field(i)] = data; + } + } +} + +} // namespace upbc diff --git a/third_party/upb/upbc/message_layout.h b/third_party/upb/upbc/message_layout.h new file mode 100644 index 00000000000..c2446a08656 --- /dev/null +++ b/third_party/upb/upbc/message_layout.h @@ -0,0 +1,107 @@ + +#ifndef UPBC_MESSAGE_LAYOUT_H +#define UPBC_MESSAGE_LAYOUT_H + +#include "absl/base/macros.h" +#include "absl/container/flat_hash_map.h" +#include "google/protobuf/descriptor.h" + +namespace upbc { + +class MessageLayout { + public: + struct Size { + void Add(const Size& other) { + size32 += other.size32; + size64 += other.size64; + } + + void MaxFrom(const Size& other) { + size32 = std::max(size32, other.size32); + size64 = std::max(size64, other.size64); + } + + void AlignUp(const Size& align) { + size32 = Align(size32, align.size32); + size64 = Align(size64, align.size64); + } + + int64_t size32; + int64_t size64; + }; + + struct SizeAndAlign { + Size size; + Size align; + + void MaxFrom(const SizeAndAlign& other) { + size.MaxFrom(other.size); + align.MaxFrom(other.align); + } + }; + + MessageLayout(const google::protobuf::Descriptor* descriptor) { + ComputeLayout(descriptor); + } + + Size GetFieldOffset(const google::protobuf::FieldDescriptor* field) const { + return GetMapValue(field_offsets_, field); + } + + Size GetOneofCaseOffset( + const google::protobuf::OneofDescriptor* oneof) const { + return GetMapValue(oneof_case_offsets_, oneof); + } + + int GetHasbitIndex(const google::protobuf::FieldDescriptor* field) const { + return GetMapValue(hasbit_indexes_, field); + } + + Size message_size() const { return size_; } + + static bool HasHasbit(const google::protobuf::FieldDescriptor* field); + static SizeAndAlign SizeOfUnwrapped( + const google::protobuf::FieldDescriptor* field); + + private: + void ComputeLayout(const google::protobuf::Descriptor* descriptor); + void PlaceNonOneofFields(const google::protobuf::Descriptor* descriptor); + void PlaceOneofFields(const google::protobuf::Descriptor* descriptor); + Size Place(SizeAndAlign size_and_align); + + template + static V GetMapValue(const absl::flat_hash_map& map, K key) { + auto iter = map.find(key); + if (iter == map.end()) { + fprintf(stderr, "No value for field.\n"); + abort(); + } + return iter->second; + } + + static bool IsPowerOfTwo(size_t val) { + return (val & (val - 1)) == 0; + } + + static size_t Align(size_t val, size_t align) { + ABSL_ASSERT(IsPowerOfTwo(align)); + return (val + align - 1) & ~(align - 1); + } + + static SizeAndAlign SizeOf(const google::protobuf::FieldDescriptor* field); + static int64_t FieldLayoutRank( + const google::protobuf::FieldDescriptor* field); + + absl::flat_hash_map + field_offsets_; + absl::flat_hash_map + hasbit_indexes_; + absl::flat_hash_map + oneof_case_offsets_; + Size maxalign_; + Size size_; +}; + +} // namespace upbc + +#endif // UPBC_MESSAGE_LAYOUT_H diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index bd98e5c479a..0c8900ae83a 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -41,7 +41,6 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" 09745575a923640154bcf307fba8aedff47f240a third_party/protobuf (v3.7.0-rc.2-247-g09745575) e143189bf6f37b3957fb31743df6a1bcf4a8c685 third_party/protoc-gen-validate (v0.0.10) 94324803a497c8f76dbc78df393ef629d3a9f3c3 third_party/udpa (heads/master) - 931bbecbd3230ae7f22efa5d203639facc47f719 third_party/upb (heads/master) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF