From 22130484ccb81110f8617e4b7ca7f8eb5268146a Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 24 Sep 2019 10:13:22 -0700 Subject: [PATCH 01/27] Bump version to 1.24 --- BUILD | 2 +- build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index c34ae1fb43a..d9152c48c3c 100644 --- a/BUILD +++ b/BUILD @@ -81,7 +81,7 @@ g_stands_for = "ganges" core_version = "7.0.0" -version = "1.24.0-pre1" +version = "1.24.0" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index 1d541a1ff9a..52c7dcacd47 100644 --- a/build.yaml +++ b/build.yaml @@ -15,7 +15,7 @@ settings: core_version: 8.0.0 csharp_major_version: 2 g_stands_for: ganges - version: 1.24.0-pre2 + version: 1.24.0 filegroups: - name: alts_tsi headers: From 033b0fe7213426a6805fe981d8e5bee4cd00f22c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 24 Sep 2019 10:13:48 -0700 Subject: [PATCH 02/27] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 6 +++--- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 8 ++++---- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 2 +- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 30 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 42be1079067..e49b3cc07d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.24.0-pre2") +set(PACKAGE_VERSION "1.24.0") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index d246fb96020..bf58326629c 100644 --- a/Makefile +++ b/Makefile @@ -461,8 +461,8 @@ Q = @ endif CORE_VERSION = 8.0.0 -CPP_VERSION = 1.24.0-pre2 -CSHARP_VERSION = 2.24.0-pre2 +CPP_VERSION = 1.24.0 +CSHARP_VERSION = 2.24.0 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 8042699a93a..f937ea009b6 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,15 +23,15 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.24.0-pre2' - version = '0.0.9-pre2' + # version = '1.24.0' + version = '0.0.9' s.version = version s.summary = 'gRPC C++ library' s.homepage = 'https://grpc.io' s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.24.0-pre2' + grpc_version = '1.24.0' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 608c83254cc..5c36c959b30 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.24.0-pre2' + version = '1.24.0' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 81f3b5b3a80..883be8df14a 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.24.0-pre2' + version = '1.24.0' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 7cb7866a474..696eeedfdb6 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.24.0-pre2' + version = '1.24.0' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 1f508bdeae9..11d51d5dfdb 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.24.0-pre2' + version = '1.24.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 249f2d27837..76b72fc5578 100644 --- a/package.xml +++ b/package.xml @@ -13,12 +13,12 @@ 2018-01-19 - 1.24.0RC2 - 1.24.0RC2 + 1.24.0 + 1.24.0 - beta - beta + stable + stable Apache 2.0 diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index eb9421183bf..c54cd9c1565 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.24.0-pre2"; } +grpc::string Version() { return "1.24.0"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index dd04f0d8cc2..664b9c6bd69 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -38,6 +38,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.24.0-pre2"; + public const string CurrentVersion = "2.24.0"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 13d194a5d2d..1229d0d74bc 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.24.0-pre2 + 2.24.0 3.8.0 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index dc5d816cc55..679c2bf0aa0 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.24.0-pre2 +set VERSION=2.24.0 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index 5a6a8cd2315..41eb8b2e1d8 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.24.0-pre2' + v = '1.24.0' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index f0d81a6c548..70b7cc0c0e2 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.24.0-pre2' + v = '1.24.0' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 82e699b001a..4ae74d6e804 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.0-pre2" +#define GRPC_OBJC_VERSION_STRING @"1.24.0" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 837a2d44379..79203037823 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.0-pre2" +#define GRPC_OBJC_VERSION_STRING @"1.24.0" #define GRPC_C_VERSION_STRING @"8.0.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 370f3604ddc..12d6a8c005c 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.24.0RC2" +#define PHP_GRPC_VERSION "1.24.0" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 31e82e8c638..731817106a8 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.24.0rc2""" +__version__ = """1.24.0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 565aaaa75ff..2b621c37cb7 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index 122ff4c3bdd..d5bc36dd071 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 058d12cbc08..f6028641ae0 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 53f52e289f8..d2eec5d4190 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index fe42a15ec15..8e4ecc9bb87 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 27996b8c8a1..a55501347dc 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 9a0e8f28ba4..bde32e4976d 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index bf233ce7af7..56ab98e6908 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.24.0.pre2' + VERSION = '1.24.0' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index fb6fb356232..7994f45e32d 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.24.0.pre2' + VERSION = '1.24.0' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 9aae9350b54..e626cdc05f7 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.24.0rc2' +VERSION = '1.24.0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 2b22fb1ae3d..4cbb21b49a8 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.0-pre2 +PROJECT_NUMBER = 1.24.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e604124986d..6aebb6c69ac 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.0-pre2 +PROJECT_NUMBER = 1.24.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 673e0cec336f5dfa2a1ba0015eacc0c69f0fae63 Mon Sep 17 00:00:00 2001 From: Hope Casey-Allen Date: Tue, 24 Sep 2019 10:53:18 -0700 Subject: [PATCH 03/27] Fix --- src/core/lib/surface/server.cc | 4 +-- test/core/end2end/fuzzers/server_fuzzer.cc | 5 ++-- .../fuzzers/server_fuzzer_corpus/hope.bin | Bin 0 -> 407 bytes tools/run_tests/generated/tests.json | 23 ++++++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 test/core/end2end/fuzzers/server_fuzzer_corpus/hope.bin diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 2cc7e88cab4..087574f218d 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -629,7 +629,7 @@ static void start_new_rpc(grpc_call_element* elem) { for (i = 0; i <= chand->registered_method_max_probes; i++) { rm = &chand->registered_methods[(hash + i) % chand->registered_method_slots]; - if (!rm) break; + if (rm->server_registered_method == nullptr) break; if (!rm->has_host) continue; if (!grpc_slice_eq(rm->host, calld->host)) continue; if (!grpc_slice_eq(rm->method, calld->path)) continue; @@ -647,7 +647,7 @@ static void start_new_rpc(grpc_call_element* elem) { for (i = 0; i <= chand->registered_method_max_probes; i++) { rm = &chand->registered_methods[(hash + i) % chand->registered_method_slots]; - if (!rm) break; + if (rm->server_registered_method == nullptr) break; if (rm->has_host) continue; if (!grpc_slice_eq(rm->method, calld->path)) continue; if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) && diff --git a/test/core/end2end/fuzzers/server_fuzzer.cc b/test/core/end2end/fuzzers/server_fuzzer.cc index 130b58f6292..031db87dc2e 100644 --- a/test/core/end2end/fuzzers/server_fuzzer.cc +++ b/test/core/end2end/fuzzers/server_fuzzer.cc @@ -57,9 +57,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { grpc_server* server = grpc_server_create(nullptr, nullptr); grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr); grpc_server_register_completion_queue(server, cq, nullptr); - // TODO(ctiller): add registered methods (one for POST, one for PUT) - // void *registered_method = - // grpc_server_register_method(server, "/reg", NULL, 0); + // TODO(ctiller): add more registered methods (one for POST, one for PUT) + grpc_server_register_method(server, "/reg", nullptr, {}, 0); grpc_server_start(server); grpc_transport* transport = grpc_create_chttp2_transport(nullptr, mock_endpoint, false); diff --git a/test/core/end2end/fuzzers/server_fuzzer_corpus/hope.bin b/test/core/end2end/fuzzers/server_fuzzer_corpus/hope.bin new file mode 100644 index 0000000000000000000000000000000000000000..46dd6219518b803580b00d01b475ffbe8c1902d8 GIT binary patch literal 407 zcmZ8dK~BRk5ZsoeL{S4A;O+wlpiR<*Nrr=&<|LSNi0Q<C2*NfL zsf^{ZJTuz0cky^nF38<-xyZ|GlEg{;^xzoaGyspj=RMB>PKHh0OV{2?WBXM(3lvmlru3}b{ms1+fG{XjHA4P*~isk}Ct(|ElSCS|ov+&CfSqfuUD Date: Wed, 2 Oct 2019 13:10:44 -0700 Subject: [PATCH 04/27] Fix python windows link problem --- setup.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 609b18e01b5..7e65a8d2c23 100644 --- a/setup.py +++ b/setup.py @@ -147,16 +147,21 @@ EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) if EXTRA_ENV_COMPILE_ARGS is None: EXTRA_ENV_COMPILE_ARGS = ' -std=c++11' - if 'win32' in sys.platform and sys.version_info < (3, 5): - EXTRA_ENV_COMPILE_ARGS += ' -D_hypot=hypot' - # We use define flags here and don't directly add to DEFINE_MACROS below to - # ensure that the expert user/builder has a way of turning it off (via the - # envvars) without adding yet more GRPC-specific envvars. - # See https://sourceforge.net/p/mingw-w64/bugs/363/ - if '32' in platform.architecture()[0]: - EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s' + if 'win32' in sys.platform: + if sys.version_info < (3, 5): + EXTRA_ENV_COMPILE_ARGS += ' -D_hypot=hypot' + # We use define flags here and don't directly add to DEFINE_MACROS below to + # ensure that the expert user/builder has a way of turning it off (via the + # envvars) without adding yet more GRPC-specific envvars. + # See https://sourceforge.net/p/mingw-w64/bugs/363/ + if '32' in platform.architecture()[0]: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s' + else: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' else: - EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' + # We need to statically link the C++ Runtime, only the C runtime is + # available dynamically + EXTRA_ENV_COMPILE_ARGS += ' /MT' elif "linux" in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -std=gnu99 -fvisibility=hidden -fno-wrapv -fno-exceptions' elif "darwin" in sys.platform: From 577ecfd76f2fb82efeda4eea9fbe560e24769109 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 2 Oct 2019 15:45:29 -0700 Subject: [PATCH 05/27] Bump version to 1.24.1 --- BUILD | 2 +- build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index d9152c48c3c..13ce1e91a91 100644 --- a/BUILD +++ b/BUILD @@ -81,7 +81,7 @@ g_stands_for = "ganges" core_version = "7.0.0" -version = "1.24.0" +version = "1.24.1" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index 52c7dcacd47..b976be26ede 100644 --- a/build.yaml +++ b/build.yaml @@ -15,7 +15,7 @@ settings: core_version: 8.0.0 csharp_major_version: 2 g_stands_for: ganges - version: 1.24.0 + version: 1.24.1 filegroups: - name: alts_tsi headers: From 5cbf8ead38bcf3473bac62646e1d419ba95dff3c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 2 Oct 2019 15:46:44 -0700 Subject: [PATCH 06/27] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 4 ++-- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 35 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e49b3cc07d5..54dc212e6a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.24.0") +set(PACKAGE_VERSION "1.24.1") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index bf58326629c..8de7287faab 100644 --- a/Makefile +++ b/Makefile @@ -461,8 +461,8 @@ Q = @ endif CORE_VERSION = 8.0.0 -CPP_VERSION = 1.24.0 -CSHARP_VERSION = 2.24.0 +CPP_VERSION = 1.24.1 +CSHARP_VERSION = 2.24.1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index f937ea009b6..3fe09873851 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.24.0' + # version = '1.24.1' version = '0.0.9' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.24.0' + grpc_version = '1.24.1' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 5c36c959b30..5dcd830cec9 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.24.0' + version = '1.24.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 883be8df14a..813e7635164 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.24.0' + version = '1.24.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 696eeedfdb6..f5514071f02 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.24.0' + version = '1.24.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 11d51d5dfdb..45464f67a61 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.24.0' + version = '1.24.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 76b72fc5578..ad9fa2b8e58 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.24.0 - 1.24.0 + 1.24.1 + 1.24.1 stable diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index c54cd9c1565..988d92483a2 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.24.0"; } +grpc::string Version() { return "1.24.1"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index 664b9c6bd69..304a3a1696c 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "2.24.0.0"; + public const string CurrentAssemblyFileVersion = "2.24.1.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.24.0"; + public const string CurrentVersion = "2.24.1"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 1229d0d74bc..ac572cab0fd 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.24.0 + 2.24.1 3.8.0 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 679c2bf0aa0..1166601f089 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.24.0 +set VERSION=2.24.1 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index 41eb8b2e1d8..c04b2297c67 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.24.0' + v = '1.24.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 70b7cc0c0e2..d90d111fb49 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.24.0' + v = '1.24.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 4ae74d6e804..332e8df64c5 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.0" +#define GRPC_OBJC_VERSION_STRING @"1.24.1" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 79203037823..2cc70252a23 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.0" +#define GRPC_OBJC_VERSION_STRING @"1.24.1" #define GRPC_C_VERSION_STRING @"8.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index 6da1fd77d65..7054b8477b3 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.24.0", + "version": "1.24.1", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 12d6a8c005c..70709f5c890 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.24.0" +#define PHP_GRPC_VERSION "1.24.1" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 731817106a8..3157f10feec 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.24.0""" +__version__ = """1.24.1""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 2b621c37cb7..37985cb0ba4 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index d5bc36dd071..7deadf603b9 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index f6028641ae0..643458e979b 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index d2eec5d4190..1b1a63e2a7d 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index 8e4ecc9bb87..d3c180ce634 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index a55501347dc..b60937c12bf 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index bde32e4976d..78f083cb26b 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 56ab98e6908..af90e611a4d 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.24.0' + VERSION = '1.24.1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 7994f45e32d..49d4c591e26 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.24.0' + VERSION = '1.24.1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index e626cdc05f7..377502ec1db 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.24.0' +VERSION = '1.24.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 4cbb21b49a8..a203a2a691e 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.0 +PROJECT_NUMBER = 1.24.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 6aebb6c69ac..e8b4f1a9122 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.0 +PROJECT_NUMBER = 1.24.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From b912fc7d8d401bb65b3147ee77d03beaa3d46038 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 30 Sep 2019 23:07:20 -0700 Subject: [PATCH 07/27] Added libatomic to gRPC python --- setup.py | 23 +++++++++++++++++++--- tools/distrib/python/grpcio_tools/setup.py | 17 ++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 7e65a8d2c23..0591f7296e0 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,9 @@ import sysconfig import setuptools from setuptools.command import egg_info +import subprocess +from subprocess import PIPE + # Redirect the manifest template from MANIFEST.in to PYTHON-MANIFEST.in. egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in' @@ -136,6 +139,18 @@ ENABLE_CYTHON_TRACING = os.environ.get( ENABLE_DOCUMENTATION_BUILD = os.environ.get( 'GRPC_PYTHON_ENABLE_DOCUMENTATION_BUILD', False) +def check_linker_need_libatomic(): + """Test if linker on system needs libatomic. + """ + code_test = (b'#include \n' + + b'int main() { return std::atomic{}; }') + cc_test = subprocess.Popen(['cc', '-x', 'c++', '-std=c++11', '-'], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE) + cc_test.communicate(input=code_test) + return cc_test.returncode != 0 + # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. # We use these environment variables to thus get around that without locking @@ -171,15 +186,17 @@ if EXTRA_ENV_LINK_ARGS is None: EXTRA_ENV_LINK_ARGS = '' if "linux" in sys.platform or "darwin" in sys.platform: EXTRA_ENV_LINK_ARGS += ' -lpthread' + if check_linker_need_libatomic(): + EXTRA_ENV_LINK_ARGS += ' -latomic' elif "win32" in sys.platform and sys.version_info < (3, 5): msvcr = cygwinccompiler.get_msvcr()[0] # TODO(atash) sift through the GCC specs to see if libstdc++ can have any # influence on the linkage outcome on MinGW for non-C++ programs. EXTRA_ENV_LINK_ARGS += ( - ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} ' - '-static'.format(msvcr=msvcr)) + ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr}' + ' -static'.format(msvcr=msvcr)) if "linux" in sys.platform: - EXTRA_ENV_LINK_ARGS += ' -Wl,-wrap,memcpy -static-libgcc' + EXTRA_ENV_LINK_ARGS += ' -Wl,-wrap,memcpy -static-libgcc' EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index e83e08b0431..b5d5786eff0 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -29,6 +29,9 @@ import sysconfig import setuptools from setuptools.command import build_ext +import subprocess +from subprocess import PIPE + # TODO(atash) add flag to disable Cython use _PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__)) @@ -62,6 +65,18 @@ PY3 = sys.version_info.major == 3 # to have been generated by building first *with* Cython support. BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False) +def check_linker_need_libatomic(): + """Test if linker on system needs libatomic. + """ + code_test = (b'#include \n' + + b'int main() { return std::atomic{}; }') + cc_test = subprocess.Popen(['cc', '-x', 'c++', '-std=c++11', '-'], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE) + cc_test.communicate(input=code_test) + return cc_test.returncode != 0 + # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. # We use these environment variables to thus get around that without locking @@ -93,6 +108,8 @@ if EXTRA_ENV_LINK_ARGS is None: EXTRA_ENV_LINK_ARGS = '' if "linux" in sys.platform or "darwin" in sys.platform: EXTRA_ENV_LINK_ARGS += ' -lpthread' + if check_linker_need_libatomic(): + EXTRA_ENV_LINK_ARGS += ' -latomic' elif "win32" in sys.platform and sys.version_info < (3, 5): msvcr = cygwinccompiler.get_msvcr()[0] # TODO(atash) sift through the GCC specs to see if libstdc++ can have any From 4fd4e5c72195be930e1b0f225df51a22d7776fa0 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 4 Oct 2019 17:25:58 -0700 Subject: [PATCH 08/27] Reformat code --- tools/distrib/python/grpcio_tools/setup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index b5d5786eff0..57075834154 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -65,18 +65,21 @@ PY3 = sys.version_info.major == 3 # to have been generated by building first *with* Cython support. BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False) + def check_linker_need_libatomic(): """Test if linker on system needs libatomic. """ code_test = (b'#include \n' + b'int main() { return std::atomic{}; }') - cc_test = subprocess.Popen(['cc', '-x', 'c++', '-std=c++11', '-'], - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + cc_test = subprocess.Popen( + ['cc', '-x', 'c++', '-std=c++11', '-'], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE) cc_test.communicate(input=code_test) return cc_test.returncode != 0 + # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. # We use these environment variables to thus get around that without locking From 91f31ad47da6d799d53fbddec81c172633229bc3 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 4 Oct 2019 20:39:49 -0700 Subject: [PATCH 09/27] Update comment --- setup.py | 3 +-- tools/distrib/python/grpcio_tools/setup.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0591f7296e0..1c668999234 100644 --- a/setup.py +++ b/setup.py @@ -140,8 +140,7 @@ ENABLE_DOCUMENTATION_BUILD = os.environ.get( 'GRPC_PYTHON_ENABLE_DOCUMENTATION_BUILD', False) def check_linker_need_libatomic(): - """Test if linker on system needs libatomic. - """ + """Test if linker on system needs libatomic.""" code_test = (b'#include \n' + b'int main() { return std::atomic{}; }') cc_test = subprocess.Popen(['cc', '-x', 'c++', '-std=c++11', '-'], diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 57075834154..7325e4bed33 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -67,8 +67,7 @@ BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False) def check_linker_need_libatomic(): - """Test if linker on system needs libatomic. - """ + """Test if linker on system needs libatomic.""" code_test = (b'#include \n' + b'int main() { return std::atomic{}; }') cc_test = subprocess.Popen( From 9433e4462fc8a7ebfc147aa035fe2dee3f0b693a Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 8 Oct 2019 11:47:19 -0700 Subject: [PATCH 10/27] Bump version to 1.24.2 --- BUILD | 2 +- build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 13ce1e91a91..182650c5e93 100644 --- a/BUILD +++ b/BUILD @@ -81,7 +81,7 @@ g_stands_for = "ganges" core_version = "7.0.0" -version = "1.24.1" +version = "1.24.2" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index b976be26ede..c1148806ea6 100644 --- a/build.yaml +++ b/build.yaml @@ -15,7 +15,7 @@ settings: core_version: 8.0.0 csharp_major_version: 2 g_stands_for: ganges - version: 1.24.1 + version: 1.24.2 filegroups: - name: alts_tsi headers: From 938d1bed77acbd7146cc290e5e30d2c682af3f8e Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 8 Oct 2019 11:48:07 -0700 Subject: [PATCH 11/27] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 4 ++-- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 35 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54dc212e6a2..ecd7c460799 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.24.1") +set(PACKAGE_VERSION "1.24.2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 8de7287faab..ceb21b20a18 100644 --- a/Makefile +++ b/Makefile @@ -461,8 +461,8 @@ Q = @ endif CORE_VERSION = 8.0.0 -CPP_VERSION = 1.24.1 -CSHARP_VERSION = 2.24.1 +CPP_VERSION = 1.24.2 +CSHARP_VERSION = 2.24.2 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 3fe09873851..5845fd9a3c6 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.24.1' + # version = '1.24.2' version = '0.0.9' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.24.1' + grpc_version = '1.24.2' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 5dcd830cec9..78811e7ea16 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.24.1' + version = '1.24.2' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 813e7635164..b5a5f691883 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.24.1' + version = '1.24.2' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index f5514071f02..13dbd06e51d 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.24.1' + version = '1.24.2' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 45464f67a61..39c11e7aff8 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.24.1' + version = '1.24.2' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index ad9fa2b8e58..935ecb6951a 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.24.1 - 1.24.1 + 1.24.2 + 1.24.2 stable diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 988d92483a2..6a692d7c756 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.24.1"; } +grpc::string Version() { return "1.24.2"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index 304a3a1696c..0e67c9d0567 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "2.24.1.0"; + public const string CurrentAssemblyFileVersion = "2.24.2.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.24.1"; + public const string CurrentVersion = "2.24.2"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index ac572cab0fd..905f7fefe35 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.24.1 + 2.24.2 3.8.0 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 1166601f089..15c67bb1695 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.24.1 +set VERSION=2.24.2 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index c04b2297c67..3538964a3c2 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.24.1' + v = '1.24.2' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index d90d111fb49..6237d3fc24b 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.24.1' + v = '1.24.2' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 332e8df64c5..2567296f04f 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.1" +#define GRPC_OBJC_VERSION_STRING @"1.24.2" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 2cc70252a23..db16988fcda 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.1" +#define GRPC_OBJC_VERSION_STRING @"1.24.2" #define GRPC_C_VERSION_STRING @"8.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index 7054b8477b3..737028cb1b7 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.24.1", + "version": "1.24.2", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 70709f5c890..a58e5a2a67d 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.24.1" +#define PHP_GRPC_VERSION "1.24.2" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 3157f10feec..63809489efa 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.24.1""" +__version__ = """1.24.2""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 37985cb0ba4..e72c465e392 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index 7deadf603b9..c2945d4b362 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 643458e979b..e105ce583b7 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 1b1a63e2a7d..880ddbfaf62 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index d3c180ce634..809dcfa90e1 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index b60937c12bf..d88211f92d1 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 78f083cb26b..be0ae55d30f 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index af90e611a4d..b1760a0ba39 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.24.1' + VERSION = '1.24.2' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 49d4c591e26..863aa765e62 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.24.1' + VERSION = '1.24.2' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 377502ec1db..f478434ee4d 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.24.1' +VERSION = '1.24.2' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index a203a2a691e..63d6a1b1db8 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.1 +PROJECT_NUMBER = 1.24.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e8b4f1a9122..dfd6c037f9d 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.1 +PROJECT_NUMBER = 1.24.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 80123e13c89121fbde4656820ee9e135724135e4 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 8 Oct 2019 10:39:12 -0700 Subject: [PATCH 12/27] Move internal_testing to a separate subspec --- gRPC.podspec | 18 ++++++++++++++---- src/objective-c/tests/Podfile | 8 ++++---- templates/gRPC.podspec.template | 18 ++++++++++++++---- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/gRPC.podspec b/gRPC.podspec index 45464f67a61..af210b2919b 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -117,11 +117,9 @@ Pod::Spec.new do |s| 'src/objective-c/GRPCClient/GRPCCall+Cronet.h', 'src/objective-c/GRPCClient/GRPCCall+OAuth2.h', 'src/objective-c/GRPCClient/GRPCCall+Tests.h', - 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.h', - 'src/objective-c/GRPCClient/internal_testing/*.h' + 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.h' ss.private_header_files = 'src/objective-c/GRPCClient/private/GRPCCore/*.h' - ss.source_files = 'src/objective-c/GRPCClient/internal_testing/*.{h,m}', - 'src/objective-c/GRPCClient/private/GRPCCore/*.{h,m}', + ss.source_files = 'src/objective-c/GRPCClient/private/GRPCCore/*.{h,m}', 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.h', 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.m', 'src/objective-c/GRPCClient/GRPCCall+ChannelCredentials.h', @@ -170,4 +168,16 @@ Pod::Spec.new do |s| ss.tvos.deployment_target = '10.0' ss.watchos.deployment_target = '4.0' end + + s.subspec 'InternalTesting' do |ss| + ss.dependency "#{s.name}/GRPCCore", version + ss.public_header_files = 'src/objective-c/GRPCClient/internal_testing/*.h' + ss.source_files = 'src/objective-c/GRPCClient/internal_testing/*.{h,m}' + ss.header_mappings_dir = 'src/objective-c/GRPCClient' + + ss.ios.deployment_target = '7.0' + ss.osx.deployment_target = '10.9' + ss.tvos.deployment_target = '10.0' + ss.watchos.deployment_target = '4.0' + end end diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile index c83e8861e93..4d677526d3b 100644 --- a/src/objective-c/tests/Podfile +++ b/src/objective-c/tests/Podfile @@ -13,10 +13,10 @@ def grpc_deps pod 'BoringSSL-GRPC', :podspec => "#{GRPC_LOCAL_SRC}/src/objective-c", :inhibit_warnings => true - pod 'gRPC', :path => GRPC_LOCAL_SRC - pod 'gRPC-Core', :path => GRPC_LOCAL_SRC - pod 'gRPC-RxLibrary', :path => GRPC_LOCAL_SRC - pod 'gRPC-ProtoRPC', :path => GRPC_LOCAL_SRC, :inhibit_warnings => true + pod 'gRPC/InternalTesting', :path => GRPC_LOCAL_SRC + pod 'gRPC-Core', :path => GRPC_LOCAL_SRC + pod 'gRPC-RxLibrary', :path => GRPC_LOCAL_SRC + pod 'gRPC-ProtoRPC', :path => GRPC_LOCAL_SRC, :inhibit_warnings => true pod 'RemoteTest', :path => "RemoteTestClient", :inhibit_warnings => true end diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index e8afdb14062..6420045b104 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -119,11 +119,9 @@ 'src/objective-c/GRPCClient/GRPCCall+Cronet.h', 'src/objective-c/GRPCClient/GRPCCall+OAuth2.h', 'src/objective-c/GRPCClient/GRPCCall+Tests.h', - 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.h', - 'src/objective-c/GRPCClient/internal_testing/*.h' + 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.h' ss.private_header_files = 'src/objective-c/GRPCClient/private/GRPCCore/*.h' - ss.source_files = 'src/objective-c/GRPCClient/internal_testing/*.{h,m}', - 'src/objective-c/GRPCClient/private/GRPCCore/*.{h,m}', + ss.source_files = 'src/objective-c/GRPCClient/private/GRPCCore/*.{h,m}', 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.h', 'src/objective-c/GRPCClient/GRPCCall+ChannelArg.m', 'src/objective-c/GRPCClient/GRPCCall+ChannelCredentials.h', @@ -172,4 +170,16 @@ ss.tvos.deployment_target = '10.0' ss.watchos.deployment_target = '4.0' end + + s.subspec 'InternalTesting' do |ss| + ss.dependency "#{s.name}/GRPCCore", version + ss.public_header_files = 'src/objective-c/GRPCClient/internal_testing/*.h' + ss.source_files = 'src/objective-c/GRPCClient/internal_testing/*.{h,m}' + ss.header_mappings_dir = 'src/objective-c/GRPCClient' + + ss.ios.deployment_target = '7.0' + ss.osx.deployment_target = '10.9' + ss.tvos.deployment_target = '10.0' + ss.watchos.deployment_target = '4.0' + end end From e2a005c1b66b2eb0f6b5135675b2c9eeaa12bc5b Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Sep 2019 12:41:50 -0700 Subject: [PATCH 13/27] Build Python wheels with tag manylinux2010 --- .../Dockerfile | 0 .../Dockerfile | 0 .../Dockerfile | 38 ++++++++++++++++ .../Dockerfile | 38 ++++++++++++++++ tools/run_tests/artifacts/artifact_targets.py | 43 ++++++++++++------- 5 files changed, 104 insertions(+), 15 deletions(-) rename tools/dockerfile/{grpc_artifact_python_manylinux_x64 => grpc_artifact_python_manylinux1_x64}/Dockerfile (100%) rename tools/dockerfile/{grpc_artifact_python_manylinux_x86 => grpc_artifact_python_manylinux1_x86}/Dockerfile (100%) create mode 100644 tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile create mode 100644 tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile diff --git a/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux1_x64/Dockerfile similarity index 100% rename from tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile rename to tools/dockerfile/grpc_artifact_python_manylinux1_x64/Dockerfile diff --git a/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux1_x86/Dockerfile similarity index 100% rename from tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile rename to tools/dockerfile/grpc_artifact_python_manylinux1_x86/Dockerfile diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile new file mode 100644 index 00000000000..37e2da568ae --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile @@ -0,0 +1,38 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Docker file for building gRPC manylinux Python artifacts. + +FROM quay.io/pypa/manylinux2010_x86_64 + +# Update the package manager +RUN yum update -y +RUN yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc + +################################### +# Install Python build requirements +RUN /opt/python/cp27-cp27m/bin/pip install cython +RUN /opt/python/cp27-cp27mu/bin/pip install cython +RUN /opt/python/cp34-cp34m/bin/pip install cython +RUN /opt/python/cp35-cp35m/bin/pip install cython +RUN /opt/python/cp36-cp36m/bin/pip install cython +RUN /opt/python/cp37-cp37m/bin/pip install cython + +#################################################### +# Install auditwheel with fix for namespace packages +RUN git clone https://github.com/pypa/auditwheel /usr/local/src/auditwheel +RUN cd /usr/local/src/auditwheel && git checkout 2.1 +RUN /opt/python/cp36-cp36m/bin/pip install /usr/local/src/auditwheel +RUN rm /usr/local/bin/auditwheel +RUN cd /usr/local/bin && ln -s /opt/python/cp36-cp36m/bin/auditwheel diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile new file mode 100644 index 00000000000..a4acb30d09d --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile @@ -0,0 +1,38 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Docker file for building gRPC manylinux Python artifacts. + +FROM quay.io/pypa/manylinux2010_i686 + +# Update the package manager +RUN yum update -y +RUN yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc + +################################### +# Install Python build requirements +RUN /opt/python/cp27-cp27m/bin/pip install cython +RUN /opt/python/cp27-cp27mu/bin/pip install cython +RUN /opt/python/cp34-cp34m/bin/pip install cython +RUN /opt/python/cp35-cp35m/bin/pip install cython +RUN /opt/python/cp36-cp36m/bin/pip install cython +RUN /opt/python/cp37-cp37m/bin/pip install cython + +#################################################### +# Install auditwheel with fix for namespace packages +RUN git clone https://github.com/pypa/auditwheel /usr/local/src/auditwheel +RUN cd /usr/local/src/auditwheel && git checkout 2.1 +RUN /opt/python/cp36-cp36m/bin/pip install /usr/local/src/auditwheel +RUN rm /usr/local/bin/auditwheel +RUN cd /usr/local/bin && ln -s /opt/python/cp36-cp36m/bin/auditwheel diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index aa1d0c3bf22..e251e29dbf7 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -135,7 +135,7 @@ class PythonArtifact: timeout_seconds=60 * 60 * 5, docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), extra_docker_args=extra_args) - elif self.platform == 'linux': + elif 'manylinux' in self.platform: if self.arch == 'x86': environ['SETARCH_CMD'] = 'linux32' # Inside the manylinux container, the python installations are located in @@ -150,10 +150,11 @@ class PythonArtifact: environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' + return create_docker_jobspec( self.name, - 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % - self.arch, + 'tools/dockerfile/grpc_artifact_python_%s_%s' % (self.platform, + self.arch), 'tools/run_tests/artifacts/build_artifact_python.sh', environ=environ, timeout_seconds=60 * 60, @@ -360,12 +361,18 @@ def targets(): CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), CSharpExtArtifact('linux', 'android', arch_abi='x86'), CSharpExtArtifact('macos', 'ios'), - PythonArtifact('linux', 'x86', 'cp27-cp27m'), - PythonArtifact('linux', 'x86', 'cp27-cp27mu'), - PythonArtifact('linux', 'x86', 'cp34-cp34m'), - PythonArtifact('linux', 'x86', 'cp35-cp35m'), - PythonArtifact('linux', 'x86', 'cp36-cp36m'), - PythonArtifact('linux', 'x86', 'cp37-cp37m'), + PythonArtifact('manylinux1', 'x86', 'cp27-cp27m'), + PythonArtifact('manylinux1', 'x86', 'cp27-cp27mu'), + PythonArtifact('manylinux1', 'x86', 'cp34-cp34m'), + PythonArtifact('manylinux1', 'x86', 'cp35-cp35m'), + PythonArtifact('manylinux1', 'x86', 'cp36-cp36m'), + PythonArtifact('manylinux1', 'x86', 'cp37-cp37m'), + PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), + PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), + PythonArtifact('manylinux2010', 'x86', 'cp34-cp34m'), + PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), + PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), + PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), PythonArtifact('linux_extra', 'armv7', '2.7'), PythonArtifact('linux_extra', 'armv7', '3.4'), PythonArtifact('linux_extra', 'armv7', '3.5'), @@ -374,12 +381,18 @@ def targets(): PythonArtifact('linux_extra', 'armv6', '3.4'), PythonArtifact('linux_extra', 'armv6', '3.5'), PythonArtifact('linux_extra', 'armv6', '3.6'), - PythonArtifact('linux', 'x64', 'cp27-cp27m'), - PythonArtifact('linux', 'x64', 'cp27-cp27mu'), - PythonArtifact('linux', 'x64', 'cp34-cp34m'), - PythonArtifact('linux', 'x64', 'cp35-cp35m'), - PythonArtifact('linux', 'x64', 'cp36-cp36m'), - PythonArtifact('linux', 'x64', 'cp37-cp37m'), + PythonArtifact('manylinux1', 'x64', 'cp27-cp27m'), + PythonArtifact('manylinux1', 'x64', 'cp27-cp27mu'), + PythonArtifact('manylinux1', 'x64', 'cp34-cp34m'), + PythonArtifact('manylinux1', 'x64', 'cp35-cp35m'), + PythonArtifact('manylinux1', 'x64', 'cp36-cp36m'), + PythonArtifact('manylinux1', 'x64', 'cp37-cp37m'), + PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), + PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), + PythonArtifact('manylinux2010', 'x64', 'cp34-cp34m'), + PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), + PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), + PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), PythonArtifact('macos', 'x64', 'python2.7'), PythonArtifact('macos', 'x64', 'python3.4'), PythonArtifact('macos', 'x64', 'python3.5'), From 60b68b884d1475e7a671791a395fdaac64646e21 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Sep 2019 13:04:43 -0700 Subject: [PATCH 14/27] Remove manylinux2010 i686 targets --- .../Dockerfile | 38 ------------------- tools/run_tests/artifacts/artifact_targets.py | 8 +--- 2 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile deleted file mode 100644 index a4acb30d09d..00000000000 --- a/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2019 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Docker file for building gRPC manylinux Python artifacts. - -FROM quay.io/pypa/manylinux2010_i686 - -# Update the package manager -RUN yum update -y -RUN yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc - -################################### -# Install Python build requirements -RUN /opt/python/cp27-cp27m/bin/pip install cython -RUN /opt/python/cp27-cp27mu/bin/pip install cython -RUN /opt/python/cp34-cp34m/bin/pip install cython -RUN /opt/python/cp35-cp35m/bin/pip install cython -RUN /opt/python/cp36-cp36m/bin/pip install cython -RUN /opt/python/cp37-cp37m/bin/pip install cython - -#################################################### -# Install auditwheel with fix for namespace packages -RUN git clone https://github.com/pypa/auditwheel /usr/local/src/auditwheel -RUN cd /usr/local/src/auditwheel && git checkout 2.1 -RUN /opt/python/cp36-cp36m/bin/pip install /usr/local/src/auditwheel -RUN rm /usr/local/bin/auditwheel -RUN cd /usr/local/bin && ln -s /opt/python/cp36-cp36m/bin/auditwheel diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index e251e29dbf7..7f35eb19cf3 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -361,18 +361,14 @@ def targets(): CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), CSharpExtArtifact('linux', 'android', arch_abi='x86'), CSharpExtArtifact('macos', 'ios'), + # TODO(https://github.com/grpc/grpc/issues/20283) + # Add manylinux2010_x86 targets once this issue is resolved. PythonArtifact('manylinux1', 'x86', 'cp27-cp27m'), PythonArtifact('manylinux1', 'x86', 'cp27-cp27mu'), PythonArtifact('manylinux1', 'x86', 'cp34-cp34m'), PythonArtifact('manylinux1', 'x86', 'cp35-cp35m'), PythonArtifact('manylinux1', 'x86', 'cp36-cp36m'), PythonArtifact('manylinux1', 'x86', 'cp37-cp37m'), - PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), - PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), - PythonArtifact('manylinux2010', 'x86', 'cp34-cp34m'), - PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), - PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), - PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), PythonArtifact('linux_extra', 'armv7', '2.7'), PythonArtifact('linux_extra', 'armv7', '3.4'), PythonArtifact('linux_extra', 'armv7', '3.5'), From c55142a632013ec9d952a5c2a64ba6393047c0c0 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Sep 2019 13:08:25 -0700 Subject: [PATCH 15/27] Add linux label to make our CI work --- tools/run_tests/artifacts/artifact_targets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 7f35eb19cf3..762c1a32399 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -110,6 +110,8 @@ class PythonArtifact: self.arch = arch self.labels = ['artifact', 'python', platform, arch, py_version] self.py_version = py_version + if 'manylinux' in platform: + self.labels.append('linux') def pre_build_jobspecs(self): return [] From 31afe769022eb43f0525a3a9eceaf1319feb8ab5 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Sep 2019 14:20:13 -0700 Subject: [PATCH 16/27] Make the artifact check accepts both manylinux1 and manylinux2010 --- tools/run_tests/artifacts/build_artifact_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 7393b88a768..a708f2e103d 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -79,12 +79,12 @@ ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_whee if [ "$GRPC_BUILD_MANYLINUX_WHEEL" != "" ] then for wheel in dist/*.whl; do - "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep \"manylinux1 + "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep \"manylinux "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" rm "$wheel" done for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do - "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep \"manylinux1 + "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep \"manylinux "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" rm "$wheel" done From 23a2ddcd65a2718426efbd3980f927f3a147de9a Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Sep 2019 14:46:50 -0700 Subject: [PATCH 17/27] Restrict the acceptable artifact tags to all existing uploadable combinations --- tools/run_tests/artifacts/build_artifact_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index a708f2e103d..91eb21fd7ee 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -79,12 +79,12 @@ ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_whee if [ "$GRPC_BUILD_MANYLINUX_WHEEL" != "" ] then for wheel in dist/*.whl; do - "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep \"manylinux + "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w 'manylinux(1|2010)_(x86_64|i686)' "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" rm "$wheel" done for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do - "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep \"manylinux + "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w 'manylinux(1|2010)_(x86_64|i686)' "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" rm "$wheel" done From 644a1c02483459a3aa4ee5eec15c31231563bc79 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 9 Oct 2019 08:41:42 -0700 Subject: [PATCH 18/27] Update docker images for manylinux --- .../grpc_artifact_linux_x64/Dockerfile | 2 +- .../grpc_artifact_linux_x86/Dockerfile | 2 +- .../Dockerfile | 9 +----- .../Dockerfile | 11 ++----- .../Dockerfile | 10 ++---- .../Dockerfile | 32 +++++++++++++++++++ .../artifacts/build_artifact_python.sh | 4 +-- 7 files changed, 41 insertions(+), 29 deletions(-) create mode 100644 tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index b3c16951229..d0faf057679 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -84,7 +84,7 @@ RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get cl # Python AuditWheel dependencies (needed to check manylinux1 compatibility) RUN apt-get install -y python3 python3-pip -RUN pip3 install auditwheel==1.10.0 +RUN pip3 install auditwheel==2.1.1 RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index c9054653067..8738137f5b2 100644 --- a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile @@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get cl # Python AuditWheel dependencies (needed to check manylinux1 compatibility) RUN apt-get install -y python3 python3-pip -RUN pip3 install auditwheel==1.10.0 +RUN pip3 install auditwheel==2.1.1 RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/grpc_artifact_python_manylinux1_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux1_x64/Dockerfile index c36ad6bec58..1ff6aa062a2 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux1_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux1_x64/Dockerfile @@ -28,11 +28,4 @@ RUN /opt/python/cp34-cp34m/bin/pip install cython RUN /opt/python/cp35-cp35m/bin/pip install cython RUN /opt/python/cp36-cp36m/bin/pip install cython RUN /opt/python/cp37-cp37m/bin/pip install cython - -#################################################### -# Install auditwheel with fix for namespace packages -RUN git clone https://github.com/pypa/auditwheel /usr/local/src/auditwheel -RUN cd /usr/local/src/auditwheel && git checkout 2.1 -RUN /opt/python/cp36-cp36m/bin/pip install /usr/local/src/auditwheel -RUN rm /usr/local/bin/auditwheel -RUN cd /usr/local/bin && ln -s /opt/python/cp36-cp36m/bin/auditwheel +RUN /opt/python/cp38-cp38/bin/pip install cython diff --git a/tools/dockerfile/grpc_artifact_python_manylinux1_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux1_x86/Dockerfile index a33e0517ae2..18b4cb62927 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux1_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux1_x86/Dockerfile @@ -28,12 +28,5 @@ RUN /opt/python/cp34-cp34m/bin/pip install cython RUN /opt/python/cp35-cp35m/bin/pip install cython RUN /opt/python/cp36-cp36m/bin/pip install cython RUN /opt/python/cp37-cp37m/bin/pip install cython - -#################################################### -# Install auditwheel with fix for namespace packages -RUN git clone https://github.com/pypa/auditwheel /usr/local/src/auditwheel -RUN cd /usr/local/src/auditwheel && git checkout 2.1 -RUN /opt/python/cp36-cp36m/bin/pip install /usr/local/src/auditwheel -RUN rm /usr/local/bin/auditwheel -RUN cd /usr/local/bin && ln -s /opt/python/cp36-cp36m/bin/auditwheel - +RUN /opt/python/cp37-cp37m/bin/pip install cython +RUN /opt/python/cp38-cp38/bin/pip install cython diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile index 37e2da568ae..66aefcaac28 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile @@ -28,11 +28,5 @@ RUN /opt/python/cp34-cp34m/bin/pip install cython RUN /opt/python/cp35-cp35m/bin/pip install cython RUN /opt/python/cp36-cp36m/bin/pip install cython RUN /opt/python/cp37-cp37m/bin/pip install cython - -#################################################### -# Install auditwheel with fix for namespace packages -RUN git clone https://github.com/pypa/auditwheel /usr/local/src/auditwheel -RUN cd /usr/local/src/auditwheel && git checkout 2.1 -RUN /opt/python/cp36-cp36m/bin/pip install /usr/local/src/auditwheel -RUN rm /usr/local/bin/auditwheel -RUN cd /usr/local/bin && ln -s /opt/python/cp36-cp36m/bin/auditwheel +RUN /opt/python/cp37-cp37m/bin/pip install cython +RUN /opt/python/cp38-cp38/bin/pip install cython diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile new file mode 100644 index 00000000000..f55a8785182 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile @@ -0,0 +1,32 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Docker file for building gRPC manylinux Python artifacts. + +FROM quay.io/pypa/manylinux2010_i686 + +# Update the package manager +RUN yum update -y +RUN yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc + +################################### +# Install Python build requirements +RUN /opt/python/cp27-cp27m/bin/pip install cython +RUN /opt/python/cp27-cp27mu/bin/pip install cython +RUN /opt/python/cp34-cp34m/bin/pip install cython +RUN /opt/python/cp35-cp35m/bin/pip install cython +RUN /opt/python/cp36-cp36m/bin/pip install cython +RUN /opt/python/cp37-cp37m/bin/pip install cython +RUN /opt/python/cp37-cp37m/bin/pip install cython +RUN /opt/python/cp38-cp38/bin/pip install cython diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 91eb21fd7ee..1fc7fd67a6f 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -79,12 +79,12 @@ ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_whee if [ "$GRPC_BUILD_MANYLINUX_WHEEL" != "" ] then for wheel in dist/*.whl; do - "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w 'manylinux(1|2010)_(x86_64|i686)' + "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" rm "$wheel" done for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do - "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w 'manylinux(1|2010)_(x86_64|i686)' + "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" rm "$wheel" done From f52b74d5bfc6ca9013965efc99673ef8d57c3115 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 7 Oct 2019 06:01:27 -0400 Subject: [PATCH 19/27] backport #20505 to v1.24.x --- .../linux/grpc_full_performance_master.sh | 13 ++----------- .../linux/grpc_full_performance_release.sh | 13 ++----------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh index 05a014b88ad..58240ccf4b5 100755 --- a/tools/internal_ci/linux/grpc_full_performance_master.sh +++ b/tools/internal_ci/linux/grpc_full_performance_master.sh @@ -31,7 +31,7 @@ tools/run_tests/run_performance_tests.py \ || EXIT_CODE=1 # prevent pushing leftover build files to remote hosts in the next step. -git clean -fdxq -e reports +git clean -fdxq -e reports -e run_performance_tests # scalability with 32cores (and upload to a different BQ table) tools/run_tests/run_performance_tests.py \ @@ -45,15 +45,6 @@ tools/run_tests/run_performance_tests.py \ || EXIT_CODE=1 # prevent pushing leftover build files to remote hosts in the next step. -git clean -fdxq -e reports - -# selected scenarios on Windows -tools/run_tests/run_performance_tests.py \ - -l csharp \ - --category scalable \ - --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \ - --bq_result_table performance_test.performance_experiment_windows \ - --xml_report reports/windows/sponge_log.xml \ - || EXIT_CODE=1 +git clean -fdxq -e reports -e run_performance_tests exit $EXIT_CODE diff --git a/tools/internal_ci/linux/grpc_full_performance_release.sh b/tools/internal_ci/linux/grpc_full_performance_release.sh index a31cea281fe..25d27c95197 100755 --- a/tools/internal_ci/linux/grpc_full_performance_release.sh +++ b/tools/internal_ci/linux/grpc_full_performance_release.sh @@ -31,7 +31,7 @@ tools/run_tests/run_performance_tests.py \ || EXIT_CODE=1 # prevent pushing leftover build files to remote hosts in the next step. -git clean -fdxq -e reports +git clean -fdxq -e reports -e run_performance_tests # scalability with 32cores (and upload to a different BQ table) tools/run_tests/run_performance_tests.py \ @@ -45,15 +45,6 @@ tools/run_tests/run_performance_tests.py \ || EXIT_CODE=1 # prevent pushing leftover build files to remote hosts in the next step. -git clean -fdxq -e reports - -# selected scenarios on Windows -tools/run_tests/run_performance_tests.py \ - -l csharp \ - --category scalable \ - --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \ - --bq_result_table performance_released.performance_experiment_windows \ - --xml_report reports/windows/sponge_log.xml \ - || EXIT_CODE=1 +git clean -fdxq -e reports -e run_performance_tests exit $EXIT_CODE From 525e0fdef21b721e86e3a470587fb9de8a03524d Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 17 Oct 2019 11:34:47 -0700 Subject: [PATCH 20/27] Remove upb submodule --- .gitmodules | 3 --- third_party/upb | 1 - tools/run_tests/sanity/check_submodules.sh | 1 - 3 files changed, 5 deletions(-) delete mode 160000 third_party/upb diff --git a/.gitmodules b/.gitmodules index 2b085e15059..f3653b475c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -51,9 +51,6 @@ [submodule "third_party/protoc-gen-validate"] path = third_party/protoc-gen-validate url = https://github.com/envoyproxy/protoc-gen-validate.git -[submodule "third_party/upb"] - path = third_party/upb - url = https://github.com/protocolbuffers/upb.git [submodule "third_party/udpa"] path = third_party/udpa url = https://github.com/cncf/udpa.git diff --git a/third_party/upb b/third_party/upb deleted file mode 160000 index 931bbecbd32..00000000000 --- a/third_party/upb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 931bbecbd3230ae7f22efa5d203639facc47f719 diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index bd98e5c479a..0c8900ae83a 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -41,7 +41,6 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" 09745575a923640154bcf307fba8aedff47f240a third_party/protobuf (v3.7.0-rc.2-247-g09745575) e143189bf6f37b3957fb31743df6a1bcf4a8c685 third_party/protoc-gen-validate (v0.0.10) 94324803a497c8f76dbc78df393ef629d3a9f3c3 third_party/udpa (heads/master) - 931bbecbd3230ae7f22efa5d203639facc47f719 third_party/upb (heads/master) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF From 21df81dfc2ddfb77c6fe48afea2801a152a03769 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 21 Oct 2019 17:25:28 -0700 Subject: [PATCH 21/27] Squashed 'third_party/upb/' content from commit 9effcbcb27 git-subtree-dir: third_party/upb git-subtree-split: 9effcbcb27f0a665f9f345030188c0b291e32482 --- .bazelci/presubmit.yml | 10 + .gitignore | 4 + .gitmodules | 3 + BUILD | 727 ++++ CMakeLists.txt | 147 + CONTRIBUTING.md | 7 + DESIGN.md | 72 + LICENSE | 26 + README.md | 134 + WORKSPACE | 39 + bazel/BUILD | 0 bazel/build_defs.bzl | 221 ++ bazel/lua.BUILD | 102 + bazel/ragel.BUILD | 193 + bazel/repository_defs.bzl | 15 + bazel/upb_proto_library.bzl | 299 ++ bazel/workspace_deps.bzl | 36 + examples/bazel/BUILD | 18 + examples/bazel/WORKSPACE | 14 + examples/bazel/foo.proto | 7 + examples/bazel/test_binary.c | 17 + .../google/protobuf/descriptor.upb.c | 485 +++ .../google/protobuf/descriptor.upb.h | 1690 ++++++++ generated_for_cmake/upb/json/parser.c | 3454 +++++++++++++++++ kokoro/ubuntu/build.sh | 16 + kokoro/ubuntu/continuous.cfg | 2 + kokoro/ubuntu/presubmit.cfg | 2 + tests/benchmark.cc | 36 + tests/bindings/googlepb/test_vs_proto2.cc | 165 + tests/bindings/lua/test_upb.lua | 750 ++++ tests/bindings/lua/test_upb.pb.lua | 80 + tests/bindings/ruby/upb.rb | 62 + tests/conformance_upb.c | 179 + tests/conformance_upb_failures.txt | 1 + tests/corpus/README | 1 + tests/corpus/temp.cc | 1 + tests/file_descriptor_parsenew_fuzzer.cc | 15 + tests/google_message1.dat | Bin 0 -> 228 bytes tests/google_message2.dat | Bin 0 -> 84570 bytes tests/google_messages.proto | 149 + tests/json/enum_from_separate_file.proto | 9 + tests/json/test.proto | 47 + tests/json/test.proto.pb | Bin 0 -> 1958 bytes tests/json/test_json.cc | 256 ++ tests/pb/test_decoder.cc | 1203 ++++++ tests/pb/test_decoder.proto | 128 + tests/pb/test_encoder.cc | 48 + tests/pb/test_varint.c | 117 + tests/test.proto | 68 + tests/test.proto.pb | Bin 0 -> 652 bytes tests/test_cpp.cc | 957 +++++ tests/test_cpp.proto | 12 + tests/test_table.cc | 679 ++++ tests/test_util.h | 230 ++ tests/testmain.cc | 16 + tests/upb_test.h | 53 + third_party/lunit/LICENSE | 32 + third_party/lunit/README.google | 9 + third_party/lunit/console.lua | 156 + third_party/lunit/lunit.lua | 725 ++++ tools/amalgamate.py | 81 + tools/make_cmakelists.py | 279 ++ tools/staleness_test.py | 30 + tools/staleness_test_lib.py | 158 + upb/bindings/README | 5 + upb/bindings/lua/def.c | 766 ++++ upb/bindings/lua/msg.c | 1060 +++++ upb/bindings/lua/upb.c | 245 ++ upb/bindings/lua/upb.h | 127 + upb/bindings/lua/upb.lua | 172 + upb/bindings/lua/upb/pb.c | 56 + upb/bindings/lua/upb/pb.lua | 3 + upb/bindings/stdc++/string.h | 69 + upb/decode.c | 604 +++ upb/decode.h | 21 + upb/def.c | 1756 +++++++++ upb/def.h | 909 +++++ upb/encode.c | 378 ++ upb/encode.h | 21 + upb/generated_util.h | 105 + upb/handlers-inl.h | 923 +++++ upb/handlers.c | 567 +++ upb/handlers.h | 732 ++++ upb/json/parser.h | 140 + upb/json/parser.rl | 3017 ++++++++++++++ upb/json/printer.c | 1406 +++++++ upb/json/printer.h | 72 + upb/legacy_msg_reflection.c | 399 ++ upb/legacy_msg_reflection.h | 191 + upb/msg.c | 111 + upb/msg.h | 69 + upb/msgfactory.c | 248 ++ upb/msgfactory.h | 48 + upb/pb/compile_decoder.c | 919 +++++ upb/pb/decoder.c | 1050 +++++ upb/pb/decoder.h | 240 ++ upb/pb/decoder.int.h | 288 ++ upb/pb/encoder.c | 570 +++ upb/pb/encoder.h | 83 + upb/pb/make-gdb-script.rb | 36 + upb/pb/textprinter.c | 340 ++ upb/pb/textprinter.h | 69 + upb/pb/varint.c | 74 + upb/pb/varint.int.h | 164 + upb/port.c | 27 + upb/port_def.inc | 152 + upb/port_undef.inc | 21 + upb/sink.c | 17 + upb/sink.h | 516 +++ upb/table.c | 912 +++++ upb/table.int.h | 507 +++ upb/upb.c | 261 ++ upb/upb.h | 364 ++ upbc/generator.cc | 826 ++++ upbc/generator.h | 12 + upbc/main.cc | 9 + upbc/message_layout.cc | 179 + upbc/message_layout.h | 107 + 118 files changed, 36435 insertions(+) create mode 100644 .bazelci/presubmit.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 BUILD create mode 100644 CMakeLists.txt create mode 100644 CONTRIBUTING.md create mode 100644 DESIGN.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 WORKSPACE create mode 100644 bazel/BUILD create mode 100644 bazel/build_defs.bzl create mode 100644 bazel/lua.BUILD create mode 100644 bazel/ragel.BUILD create mode 100644 bazel/repository_defs.bzl create mode 100644 bazel/upb_proto_library.bzl create mode 100644 bazel/workspace_deps.bzl create mode 100644 examples/bazel/BUILD create mode 100644 examples/bazel/WORKSPACE create mode 100644 examples/bazel/foo.proto create mode 100644 examples/bazel/test_binary.c create mode 100644 generated_for_cmake/google/protobuf/descriptor.upb.c create mode 100644 generated_for_cmake/google/protobuf/descriptor.upb.h create mode 100644 generated_for_cmake/upb/json/parser.c create mode 100644 kokoro/ubuntu/build.sh create mode 100644 kokoro/ubuntu/continuous.cfg create mode 100644 kokoro/ubuntu/presubmit.cfg create mode 100644 tests/benchmark.cc create mode 100644 tests/bindings/googlepb/test_vs_proto2.cc create mode 100644 tests/bindings/lua/test_upb.lua create mode 100644 tests/bindings/lua/test_upb.pb.lua create mode 100644 tests/bindings/ruby/upb.rb create mode 100644 tests/conformance_upb.c create mode 100644 tests/conformance_upb_failures.txt create mode 100644 tests/corpus/README create mode 100644 tests/corpus/temp.cc create mode 100644 tests/file_descriptor_parsenew_fuzzer.cc create mode 100644 tests/google_message1.dat create mode 100644 tests/google_message2.dat create mode 100644 tests/google_messages.proto create mode 100644 tests/json/enum_from_separate_file.proto create mode 100644 tests/json/test.proto create mode 100644 tests/json/test.proto.pb create mode 100644 tests/json/test_json.cc create mode 100644 tests/pb/test_decoder.cc create mode 100644 tests/pb/test_decoder.proto create mode 100644 tests/pb/test_encoder.cc create mode 100644 tests/pb/test_varint.c create mode 100644 tests/test.proto create mode 100644 tests/test.proto.pb create mode 100644 tests/test_cpp.cc create mode 100644 tests/test_cpp.proto create mode 100644 tests/test_table.cc create mode 100644 tests/test_util.h create mode 100644 tests/testmain.cc create mode 100644 tests/upb_test.h create mode 100644 third_party/lunit/LICENSE create mode 100644 third_party/lunit/README.google create mode 100644 third_party/lunit/console.lua create mode 100644 third_party/lunit/lunit.lua create mode 100755 tools/amalgamate.py create mode 100755 tools/make_cmakelists.py create mode 100644 tools/staleness_test.py create mode 100644 tools/staleness_test_lib.py create mode 100644 upb/bindings/README create mode 100644 upb/bindings/lua/def.c create mode 100644 upb/bindings/lua/msg.c create mode 100644 upb/bindings/lua/upb.c create mode 100644 upb/bindings/lua/upb.h create mode 100644 upb/bindings/lua/upb.lua create mode 100644 upb/bindings/lua/upb/pb.c create mode 100644 upb/bindings/lua/upb/pb.lua create mode 100644 upb/bindings/stdc++/string.h create mode 100644 upb/decode.c create mode 100644 upb/decode.h create mode 100644 upb/def.c create mode 100644 upb/def.h create mode 100644 upb/encode.c create mode 100644 upb/encode.h create mode 100644 upb/generated_util.h create mode 100644 upb/handlers-inl.h create mode 100644 upb/handlers.c create mode 100644 upb/handlers.h create mode 100644 upb/json/parser.h create mode 100644 upb/json/parser.rl create mode 100644 upb/json/printer.c create mode 100644 upb/json/printer.h create mode 100644 upb/legacy_msg_reflection.c create mode 100644 upb/legacy_msg_reflection.h create mode 100644 upb/msg.c create mode 100644 upb/msg.h create mode 100644 upb/msgfactory.c create mode 100644 upb/msgfactory.h create mode 100644 upb/pb/compile_decoder.c create mode 100644 upb/pb/decoder.c create mode 100644 upb/pb/decoder.h create mode 100644 upb/pb/decoder.int.h create mode 100644 upb/pb/encoder.c create mode 100644 upb/pb/encoder.h create mode 100755 upb/pb/make-gdb-script.rb create mode 100644 upb/pb/textprinter.c create mode 100644 upb/pb/textprinter.h create mode 100644 upb/pb/varint.c create mode 100644 upb/pb/varint.int.h create mode 100644 upb/port.c create mode 100644 upb/port_def.inc create mode 100644 upb/port_undef.inc create mode 100644 upb/sink.c create mode 100644 upb/sink.h create mode 100644 upb/table.c create mode 100644 upb/table.int.h create mode 100644 upb/upb.c create mode 100644 upb/upb.h create mode 100644 upbc/generator.cc create mode 100644 upbc/generator.h create mode 100644 upbc/main.cc create mode 100644 upbc/message_layout.cc create mode 100644 upbc/message_layout.h diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml new file mode 100644 index 00000000000..38ca79304f9 --- /dev/null +++ b/.bazelci/presubmit.yml @@ -0,0 +1,10 @@ +--- +tasks: + ubuntu: + platform: ubuntu1604 + test_targets: + - //... + macos: + platform: macos + test_targets: + - //... diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..c9bd5becf81 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.s?? +obj/ +lib/ +bazel-* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..8b52c1d2165 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/protobuf"] + path = third_party/protobuf + url = https://github.com/google/protobuf.git diff --git a/BUILD b/BUILD new file mode 100644 index 00000000000..ad85b202a9a --- /dev/null +++ b/BUILD @@ -0,0 +1,727 @@ +load( + "//bazel:build_defs.bzl", + "generated_file_staleness_test", + "licenses", # copybara:strip_for_google3 + "lua_binary", + "lua_cclibrary", + "lua_library", + "lua_test", + "make_shell_script", + "upb_amalgamation", +) +load( + "//bazel:upb_proto_library.bzl", + "upb_proto_library", + "upb_proto_reflection_library", +) + +licenses(["notice"]) # BSD (Google-authored w/ possible external contributions) + +exports_files([ + "LICENSE", + "build_defs", +]) + +CPPOPTS = [ + # copybara:strip_for_google3_begin + "-Werror", + "-Wno-long-long", + # copybara:strip_end +] + +COPTS = CPPOPTS + [ + # copybara:strip_for_google3_begin + "-pedantic", + "-Wstrict-prototypes", + # copybara:strip_end +] + +config_setting( + name = "darwin", + values = {"cpu": "darwin"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "windows", + constraint_values = ["@bazel_tools//platforms:windows"], +) + +config_setting( + name = "fuzz", + values = {"define": "fuzz=true"}, +) + +# Public C/C++ libraries ####################################################### + +cc_library( + name = "upb", + srcs = [ + "upb/decode.c", + "upb/encode.c", + "upb/generated_util.h", + "upb/msg.c", + "upb/msg.h", + "upb/port.c", + "upb/port_def.inc", + "upb/port_undef.inc", + "upb/table.c", + "upb/table.int.h", + "upb/upb.c", + ], + hdrs = [ + "upb/decode.h", + "upb/encode.h", + "upb/upb.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + visibility = ["//visibility:public"], +) + +# Common support routines used by generated code. This library has no +# implementation, but depends on :upb and exposes a few more hdrs. +# +# This is public only because we have no way of visibility-limiting it to +# upb_proto_library() only. This interface is not stable and by using it you +# give up any backward compatibility guarantees. +cc_library( + name = "generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + hdrs = [ + "upb/generated_util.h", + "upb/msg.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + textual_hdrs = [ + "upb/port_def.inc", + "upb/port_undef.inc", + ], + visibility = ["//visibility:public"], + deps = [":upb"], +) + +upb_proto_library( + name = "descriptor_upbproto", + visibility = ["//visibility:public"], + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +cc_library( + name = "reflection", + srcs = [ + "upb/def.c", + "upb/msgfactory.c", + ], + hdrs = [ + "upb/def.h", + "upb/msgfactory.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + visibility = ["//visibility:public"], + deps = [ + ":descriptor_upbproto", + ":table", + ":upb", + ], +) + +# Internal C/C++ libraries ##################################################### + +cc_library( + name = "table", + hdrs = ["upb/table.int.h"], + deps = [":upb"], +) + +# Legacy C/C++ Libraries (not recommended for new code) ######################## + +cc_library( + name = "legacy_msg_reflection", + srcs = [ + "upb/legacy_msg_reflection.c", + ], + hdrs = ["upb/legacy_msg_reflection.h"], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":table", + ":upb", + ], +) + +cc_library( + name = "handlers", + srcs = [ + "upb/handlers.c", + "upb/handlers-inl.h", + "upb/sink.c", + ], + hdrs = [ + "upb/handlers.h", + "upb/sink.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":reflection", + ":table", + ":upb", + ], +) + +cc_library( + name = "upb_pb", + srcs = [ + "upb/pb/compile_decoder.c", + "upb/pb/decoder.c", + "upb/pb/decoder.int.h", + "upb/pb/encoder.c", + "upb/pb/textprinter.c", + "upb/pb/varint.c", + "upb/pb/varint.int.h", + ], + hdrs = [ + "upb/pb/decoder.h", + "upb/pb/encoder.h", + "upb/pb/textprinter.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":descriptor_upbproto", + ":handlers", + ":reflection", + ":table", + ":upb", + ], +) + +# copybara:strip_for_google3_begin +cc_library( + name = "upb_json", + srcs = [ + "upb/json/parser.c", + "upb/json/printer.c", + ], + hdrs = [ + "upb/json/parser.h", + "upb/json/printer.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":upb", + ":upb_pb", + ], +) +# copybara:strip_end + +cc_library( + name = "upb_cc_bindings", + hdrs = [ + "upb/bindings/stdc++/string.h", + ], + deps = [ + ":descriptor_upbproto", + ":handlers", + ":upb", + ], +) + +# upb compiler ################################################################# + +cc_library( + name = "upbc_generator", + srcs = [ + "upbc/generator.cc", + "upbc/message_layout.cc", + "upbc/message_layout.h", + ], + hdrs = ["upbc/generator.h"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + "@absl//absl/base:core_headers", + "@absl//absl/container:flat_hash_map", + "@absl//absl/strings", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:protoc_lib", + ], +) + +cc_binary( + name = "protoc-gen-upb", + srcs = ["upbc/main.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + visibility = ["//visibility:public"], + deps = [ + ":upbc_generator", + "@com_google_protobuf//:protoc_lib", + ], +) + +# We strip the tests and remaining rules from google3 until the upb_proto_library() +# and upb_proto_reflection_library() rules are fixed. + +# C/C++ tests ################################################################## + +cc_binary( + name = "benchmark", + testonly = 1, + srcs = ["tests/benchmark.cc"], + deps = [ + ":descriptor_upbproto", + ":descriptor_upbreflection", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "upb_test", + testonly = 1, + srcs = [ + "tests/testmain.cc", + ], + hdrs = [ + "tests/test_util.h", + "tests/upb_test.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":handlers", + ":upb", + ], +) + +cc_test( + name = "test_varint", + srcs = [ + "tests/pb/test_varint.c", + "upb/pb/varint.int.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), + deps = [ + ":upb", + ":upb_pb", + ":upb_test", + ], +) + +proto_library( + name = "test_decoder_proto", + srcs = [ + "tests/pb/test_decoder.proto", + ], +) + +upb_proto_reflection_library( + name = "test_decoder_upbproto", + deps = [":test_decoder_proto"], +) + +cc_test( + name = "test_decoder", + srcs = [ + "tests/pb/test_decoder.cc", + "upb/pb/varint.int.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":handlers", + ":test_decoder_upbproto", + ":upb", + ":upb_pb", + ":upb_test", + ], +) + +proto_library( + name = "test_cpp_proto", + srcs = [ + "tests/test_cpp.proto", + ], +) + +upb_proto_reflection_library( + name = "test_cpp_upbproto", + deps = ["test_cpp_proto"], +) + +cc_test( + name = "test_cpp", + srcs = ["tests/test_cpp.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":handlers", + ":reflection", + ":test_cpp_upbproto", + ":upb", + ":upb_pb", + ":upb_test", + ], +) + +cc_test( + name = "test_table", + srcs = ["tests/test_table.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":table", + ":upb", + ":upb_test", + ], +) + +# OSS-Fuzz test +cc_binary( + name = "file_descriptor_parsenew_fuzzer", + testonly = 1, + srcs = ["tests/file_descriptor_parsenew_fuzzer.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }) + select({ + "//conditions:default": [], + ":fuzz": ["-fsanitize=fuzzer,address"], + }), + defines = select({ + "//conditions:default": [], + ":fuzz": ["HAVE_FUZZER"], + }), + deps = [ + ":descriptor_upbproto", + ":upb", + ], +) + +# copybara:strip_for_google3_begin +upb_proto_reflection_library( + name = "descriptor_upbreflection", + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +cc_test( + name = "test_encoder", + srcs = ["tests/pb/test_encoder.cc"], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":descriptor_upbproto", + ":descriptor_upbreflection", + ":upb", + ":upb_cc_bindings", + ":upb_pb", + ":upb_test", + ], +) + +proto_library( + name = "test_json_enum_from_separate", + srcs = ["tests/json/enum_from_separate_file.proto"], + deps = [":test_json_proto"], +) + +proto_library( + name = "test_json_proto", + srcs = ["tests/json/test.proto"], +) + +upb_proto_reflection_library( + name = "test_json_upbprotoreflection", + deps = ["test_json_proto"], +) + +upb_proto_library( + name = "test_json_enum_from_separate_upbproto", + deps = [":test_json_enum_from_separate"], +) + +upb_proto_library( + name = "test_json_upbproto", + deps = [":test_json_proto"], +) + +cc_test( + name = "test_json", + srcs = [ + "tests/json/test_json.cc", + ], + copts = select({ + ":windows": [], + "//conditions:default": CPPOPTS + }), + deps = [ + ":test_json_upbproto", + ":test_json_upbprotoreflection", + ":upb_json", + ":upb_test", + ], +) +# copybara:strip_end + +upb_proto_library( + name = "conformance_proto_upb", + testonly = 1, + deps = ["@com_google_protobuf//:conformance_proto"], +) + +upb_proto_library( + name = "test_messages_proto3_proto_upb", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto3_proto"], +) + +cc_binary( + name = "conformance_upb", + testonly = 1, + srcs = [ + "tests/conformance_upb.c", + ], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }) + ["-Ibazel-out/k8-fastbuild/bin"], + deps = [ + ":conformance_proto_upb", + ":test_messages_proto3_proto_upb", + ":upb", + ], +) + +make_shell_script( + name = "gen_test_conformance_upb", + out = "test_conformance_upb.sh", + contents = "external/com_google_protobuf/conformance_test_runner ./conformance_upb", +) + +sh_test( + name = "test_conformance_upb", + srcs = ["test_conformance_upb.sh"], + data = [ + "tests/conformance_upb_failures.txt", + ":conformance_upb", + "@com_google_protobuf//:conformance_test_runner", + ], +) + +# copybara:strip_for_google3_begin + +# Amalgamation ################################################################# + +py_binary( + name = "amalgamate", + srcs = ["tools/amalgamate.py"], +) + +upb_amalgamation( + name = "gen_amalgamation", + outs = [ + "upb.c", + "upb.h", + ], + amalgamator = ":amalgamate", + libs = [ + ":upb", + ":descriptor_upbproto", + ":reflection", + ":handlers", + ":upb_pb", + ":upb_json", + ], +) + +cc_library( + name = "amalgamation", + srcs = ["upb.c"], + hdrs = ["upb.h"], + copts = select({ + ":windows": [], + "//conditions:default": COPTS + }), +) + +# Lua libraries. ############################################################### + +lua_cclibrary( + name = "lua/upb_c", + srcs = [ + "upb/bindings/lua/def.c", + "upb/bindings/lua/msg.c", + "upb/bindings/lua/upb.c", + ], + hdrs = [ + "upb/bindings/lua/upb.h", + ], + deps = [ + "legacy_msg_reflection", + "upb", + "upb_pb", + ], +) + +lua_library( + name = "lua/upb", + srcs = ["upb/bindings/lua/upb.lua"], + luadeps = ["lua/upb_c"], + strip_prefix = "upb/bindings/lua", +) + +lua_cclibrary( + name = "lua/upb/pb_c", + srcs = ["upb/bindings/lua/upb/pb.c"], + luadeps = ["lua/upb_c"], + deps = ["upb_pb"], +) + +lua_library( + name = "lua/upb/pb", + srcs = ["upb/bindings/lua/upb/pb.lua"], + luadeps = [ + "lua/upb", + "lua/upb/pb_c", + ], + strip_prefix = "upb/bindings/lua", +) + +# Lua tests. ################################################################### + +lua_test( + name = "lua/test_upb", + luadeps = ["lua/upb"], + luamain = "tests/bindings/lua/test_upb.lua", +) + +lua_test( + name = "lua/test_upb_pb", + luadeps = ["lua/upb/pb"], + luamain = "tests/bindings/lua/test_upb.pb.lua", +) + +# Test the CMake build ######################################################### + +filegroup( + name = "cmake_files", + srcs = glob([ + "CMakeLists.txt", + "generated_for_cmake/**/*", + "google/**/*", + "upbc/**/*", + "upb/**/*", + "tests/**/*", + ]), +) + +make_shell_script( + name = "gen_run_cmake_build", + out = "run_cmake_build.sh", + contents = "find . && mkdir build && cd build && cmake .. && make -j8 && make test", +) + +sh_test( + name = "cmake_build", + srcs = ["run_cmake_build.sh"], + data = [":cmake_files"], +) + +# Generated files ############################################################## + +exports_files(["tools/staleness_test.py"]) + +py_library( + name = "staleness_test_lib", + testonly = 1, + srcs = ["tools/staleness_test_lib.py"], +) + +py_binary( + name = "make_cmakelists", + srcs = ["tools/make_cmakelists.py"], +) + +genrule( + name = "gen_cmakelists", + srcs = [ + "BUILD", + "WORKSPACE", + ":cmake_files", + ], + outs = ["generated-in/CMakeLists.txt"], + cmd = "$(location :make_cmakelists) $@", + tools = [":make_cmakelists"], +) + +genrule( + name = "generate_json_ragel", + srcs = ["upb/json/parser.rl"], + outs = ["upb/json/parser.c"], + cmd = "$(location @ragel//:ragelc) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", + tools = ["@ragel//:ragelc"], +) + +genrule( + name = "copy_json_ragel", + srcs = ["upb/json/parser.c"], + outs = ["generated-in/generated_for_cmake/upb/json/parser.c"], + cmd = "cp $< $@", +) + +genrule( + name = "copy_protos", + srcs = [":descriptor_upbproto"], + outs = [ + "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.c", + "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.h", + ], + cmd = "cp $(SRCS) $(@D)/generated-in/generated_for_cmake/google/protobuf", +) + +generated_file_staleness_test( + name = "test_generated_files", + outs = [ + "CMakeLists.txt", + "generated_for_cmake/google/protobuf/descriptor.upb.c", + "generated_for_cmake/google/protobuf/descriptor.upb.h", + "generated_for_cmake/upb/json/parser.c", + ], + generated_pattern = "generated-in/%s", +) + +# copybara:strip_end diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..836c5ff1fe5 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,147 @@ +# This file was generated from BUILD using tools/make_cmakelists.py. + +cmake_minimum_required(VERSION 3.1) + +if(${CMAKE_VERSION} VERSION_LESS 3.12) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.12) +endif() + +cmake_minimum_required (VERSION 3.0) +cmake_policy(SET CMP0048 NEW) + +project(upb) + + +# Prevent CMake from setting -rdynamic on Linux (!!). +SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# Set default build type. +if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() + +# When using Ninja, compiler output won't be colorized without this. +include(CheckCXXCompilerFlag) +CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS) +if(SUPPORTS_COLOR_ALWAYS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") +endif() + +# Implement ASAN/UBSAN options +if(UPB_ENABLE_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") +endif() + +if(UPB_ENABLE_UBSAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") +endif() + +include_directories(.) +include_directories(generated_for_cmake) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +if(APPLE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") +elseif(UNIX) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") +endif() + +enable_testing() + +add_library(upb + upb/decode.c + upb/encode.c + upb/generated_util.h + upb/msg.c + upb/msg.h + upb/port.c + upb/port_def.inc + upb/port_undef.inc + upb/table.c + upb/table.int.h + upb/upb.c + upb/decode.h + upb/encode.h + upb/upb.h) +add_library(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE) +target_link_libraries(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE + upb) +add_library(reflection + upb/def.c + upb/msgfactory.c + upb/def.h + upb/msgfactory.h) +target_link_libraries(reflection + descriptor_upbproto + table + upb) +add_library(table INTERFACE) +target_link_libraries(table INTERFACE + upb) +add_library(legacy_msg_reflection + upb/legacy_msg_reflection.c + upb/legacy_msg_reflection.h) +target_link_libraries(legacy_msg_reflection + table + upb) +add_library(handlers + upb/handlers.c + upb/handlers-inl.h + upb/sink.c + upb/handlers.h + upb/sink.h) +target_link_libraries(handlers + reflection + table + upb) +add_library(upb_pb + upb/pb/compile_decoder.c + upb/pb/decoder.c + upb/pb/decoder.int.h + upb/pb/encoder.c + upb/pb/textprinter.c + upb/pb/varint.c + upb/pb/varint.int.h + upb/pb/decoder.h + upb/pb/encoder.h + upb/pb/textprinter.h) +target_link_libraries(upb_pb + descriptor_upbproto + handlers + reflection + table + upb) +add_library(upb_json + generated_for_cmake/upb/json/parser.c + upb/json/printer.c + upb/json/parser.h + upb/json/printer.h) +target_link_libraries(upb_json + upb + upb_pb) +add_library(upb_cc_bindings INTERFACE) +target_link_libraries(upb_cc_bindings INTERFACE + descriptor_upbproto + handlers + upb) +add_library(upb_test + tests/testmain.cc + tests/test_util.h + tests/upb_test.h) +target_link_libraries(upb_test + handlers + upb) + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..2f866b4e291 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,7 @@ +## Signing the CLA + +Please sign the [Google Contributor License Agreement +(CLA)](https://cla.developers.google.com/) +before sending pull requests. For any code changes to be +accepted, the CLA must be signed. It's a quick process, I +promise! diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000000..a7a8a284516 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,72 @@ + +μpb Design +---------- + +μpb has the following design goals: + +- C89 compatible. +- small code size (both for the core library and generated messages). +- fast performance (hundreds of MB/s). +- idiomatic for C programs. +- easy to wrap in high-level languages (Python, Ruby, Lua, etc) with + good performance and all standard protobuf features. +- hands-off about memory management, allowing for easy integration + with existing VMs and/or garbage collectors. +- offers binary ABI compatibility between apps, generated messages, and + the core library (doesn't require re-generating messages or recompiling + your application when the core library changes). +- provides all features that users expect from a protobuf library + (generated messages in C, reflection, text format, etc.). +- layered, so the core is small and doesn't require descriptors. +- tidy about symbol references, so that any messages or features that + aren't used by a C program can have their code GC'd by the linker. +- possible to use protobuf binary format without leaking message/field + names into the binary. + +μpb accomplishes these goals by keeping a very small core that does not contain +descriptors. We need some way of knowing what fields are in each message and +where they live, but instead of descriptors, we keep a small/lightweight summary +of the .proto file. We call this a `upb_msglayout`. It contains the bare +minimum of what we need to know to parse and serialize protobuf binary format +into our internal representation for messages, `upb_msg`. + +The core then contains functions to parse/serialize a message, given a `upb_msg*` +and a `const upb_msglayout*`. + +This approach is similar to [nanopb](https://github.com/nanopb/nanopb) which +also compiles message definitions to a compact, internal representation without +names. However nanopb does not aim to be a fully-featured library, and has no +support for text format, JSON, or descriptors. μpb is unique in that it has a +small core similar to nanopb (though not quite as small), but also offers a +full-featured protobuf library for applications that want reflection, text +format, JSON format, etc. + +Without descriptors, the core doesn't have access to field names, so it cannot +parse/serialize to protobuf text format or JSON. Instead this functionality +lives in separate modules that depend on the module implementing descriptors. +With the descriptor module we can parse/serialize binary descriptors and +validate that they follow all the rules of protobuf schemas. + +To provide binary compatibility, we version the structs that generated messages +use to create a `upb_msglayout*`. The current initializers are +`upb_msglayout_msginit_v1`, `upb_msglayout_fieldinit_v1`, etc. Then +`upb_msglayout*` uses these as its internal representation. If upb changes its +internal representation for a `upb_msglayout*`, it will also include code to +convert the old representation to the new representation. This will use some +more memory/CPU at runtime to convert between the two, but apps that statically +link μpb will never need to worry about this. + +TODO +---- + +1. revise our generated code until it is in a state where we feel comfortable + committing to API/ABI stability for it. In particular there is an open + question of whether non-ABI-compatible field accesses should have a + fastpath different from the ABI-compatible field access. +1. Add missing features (maps, extensions, unknown fields). +1. Flesh out C++ wrappers. +1. *(lower-priority)*: revise all of the existing encoders/decoders and + handlers. We probably will want to keep handlers, since they let us decouple + encoders/decoders from `upb_msg`, but we need to simplify all of that a LOT. + Likely we will want to make handlers only per-message instead of per-field, + except for variable-length fields. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000000..da939845db6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ + +Copyright (c) 2009-2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of any other + contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY GOOGLE INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL GOOGLE INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 00000000000..a08aa0e9092 --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ + +# μpb - a small protobuf implementation in C + +|Platform|Build Status| +|--------|------------| +|macOS|[![Build Status](https://storage.googleapis.com/upb-kokoro-results/status-badge/macos.png)](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Aupb%2Fmacos%2Fcontinuous)| +|ubuntu|[![Build Status](https://storage.googleapis.com/upb-kokoro-results/status-badge/ubuntu.png)](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Aupb%2Fubuntu%2Fcontinuous)| + +μpb (often written 'upb') is a small protobuf implementation written in C. + +upb generates a C API for creating, parsing, and serializing messages +as declared in `.proto` files. upb is heavily arena-based: all +messages always live in an arena (note: the arena can live in stack or +static memory if desired). Here is a simple example: + +```c +#include "conformance/conformance.upb.h" + +void foo(const char* data, size_t size) { + upb_arena *arena; + + /* Generated message type. */ + conformance_ConformanceRequest *request; + conformance_ConformanceResponse *response; + + arena = upb_arena_new(); + request = conformance_ConformanceRequest_parse(data, size, arena); + response = conformance_ConformanceResponse_new(arena); + + switch (conformance_ConformanceRequest_payload_case(request)) { + case conformance_ConformanceRequest_payload_protobuf_payload: { + upb_strview payload = conformance_ConformanceRequest_protobuf_payload(request); + // ... + break; + } + + case conformance_ConformanceRequest_payload_NOT_SET: + fprintf(stderr, "conformance_upb: Request didn't have payload.\n"); + break; + + default: { + static const char msg[] = "Unsupported input format."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + break; + } + } + + /* Frees all messages on the arena. */ + upb_arena_free(arena); +} +``` + +API and ABI are both subject to change! Please do not distribute +as a shared library for this reason (for now at least). + +## Using upb in your project + +Currently only Bazel is supported (CMake support is partial and incomplete +but full CMake support is an eventual goal). + +To use upb in your Bazel project, first add upb to your `WORKSPACE` file, +either as a `git_repository()` or as a `new_local_repository()` with a +Git Submodule. (For an example, see `examples/bazel/ in this repo). + +```python +# Add this to your WORKSPACE file. +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "upb", + remote = "https://github.com/protocolbuffers/upb.git", + commit = "d16bf99ac4658793748cda3251226059892b3b7b", +) + +load("@upb//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() +``` + +Then in your BUILD file you can add `upb_proto_library()` rules that +generate code for a corresponding `proto_library()` rule. For +example: + +```python +# Add this to your BUILD file. +load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library") + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) + +upb_proto_library( + name = "foo_upbproto", + deps = [":foo_proto"], +) + +cc_binary( + name = "test_binary", + srcs = ["test_binary.c"], + deps = [":foo_upbproto"], +) +``` + +Then in your `.c` file you can #include the generated header: + +```c +#include "foo.upb.h" + +/* Insert code that uses generated types. */ +``` + +## Old "handlers" interfaces + +This library contains several semi-deprecated interfaces (see BUILD +file for more info about which interfaces are deprecated). These +deprecated interfaces are still used in some significant projects, +such as the Ruby and PHP C bindings for protobuf in the [main protobuf +repo](https://github.com/protocolbuffers/protobuf). The goal is to +migrate the Ruby/PHP bindings to use the newer, simpler interfaces +instead. Please do not use the old interfaces in new code. + +## Lua bindings + +This repo has some Lua bindings for the core library. These are +experimental and very incomplete. These are currently included in +order to validate that the C API is suitable for wrapping. As the +project matures these Lua bindings may become publicly available. + +## Contact + +Author: Josh Haberman ([jhaberman@gmail.com](mailto:jhaberman@gmail.com), +[haberman@google.com](mailto:haberman@google.com)) diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 00000000000..c1c8c3be9b7 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,39 @@ +workspace(name = "upb") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() + +http_archive( + name = "lua", + build_file = "//bazel:lua.BUILD", + sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b", + strip_prefix = "lua-5.2.4", + urls = [ + "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz", + "https://www.lua.org/ftp/lua-5.2.4.tar.gz", + ], +) + +http_archive( + name = "ragel", + build_file = "//bazel:ragel.BUILD", + sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", + strip_prefix = "ragel-6.10", + urls = ["http://www.colm.net/files/ragel/ragel-6.10.tar.gz"], +) + +http_archive( + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip"], # 2019-01-07 + strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb", + sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86", +) + +http_archive( + name = "com_github_google_benchmark", + urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"], + strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be", + sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3", +) diff --git a/bazel/BUILD b/bazel/BUILD new file mode 100644 index 00000000000..e69de29bb2d diff --git a/bazel/build_defs.bzl b/bazel/build_defs.bzl new file mode 100644 index 00000000000..08bb44ee3f7 --- /dev/null +++ b/bazel/build_defs.bzl @@ -0,0 +1,221 @@ +"""Internal rules for building upb.""" + +load(":upb_proto_library.bzl", "GeneratedSrcsInfo") + +def _librule(name): + return name + "_lib" + +def _get_real_short_path(file): + # For some reason, files from other archives have short paths that look like: + # ../com_google_protobuf/google/protobuf/descriptor.proto + short_path = file.short_path + if short_path.startswith("../"): + second_slash = short_path.index("/", 3) + short_path = short_path[second_slash + 1:] + return short_path + +def _get_real_root(file): + real_short_path = _get_real_short_path(file) + return file.path[:-len(real_short_path) - 1] + +def _get_real_roots(files): + roots = {} + for file in files: + real_root = _get_real_root(file) + if real_root: + roots[real_root] = True + return roots.keys() + +def lua_cclibrary(name, srcs, hdrs = [], deps = [], luadeps = []): + lib_rule = name + "_lib" + so_rule = "lib" + name + ".so" + so_file = _remove_prefix(name, "lua/") + ".so" + + native.cc_library( + name = _librule(name), + hdrs = hdrs, + srcs = srcs, + deps = deps + [_librule(dep) for dep in luadeps] + ["@lua//:liblua_headers"], + ) + + native.cc_binary( + name = so_rule, + linkshared = True, + deps = [_librule(name)], + linkopts = select({ + ":darwin": [ + "-undefined dynamic_lookup", + ], + "//conditions:default": [], + }), + ) + + native.genrule( + name = name + "_copy", + srcs = [":" + so_rule], + outs = [so_file], + cmd = "cp $< $@", + ) + + native.filegroup( + name = name, + data = [so_file], + ) + +def _remove_prefix(str, prefix): + if not str.startswith(prefix): + fail("%s doesn't start with %s" % (str, prefix)) + return str[len(prefix):] + +def _remove_suffix(str, suffix): + if not str.endswith(suffix): + fail("%s doesn't end with %s" % (str, suffix)) + return str[:-len(suffix)] + +def lua_library(name, srcs, strip_prefix, luadeps = []): + outs = [_remove_prefix(src, strip_prefix + "/") for src in srcs] + native.genrule( + name = name + "_copy", + srcs = srcs, + outs = outs, + cmd = "cp $(SRCS) $(@D)", + ) + + native.filegroup( + name = name, + data = outs + luadeps, + ) + +def make_shell_script(name, contents, out): + contents = contents.replace("$", "$$") + native.genrule( + name = "gen_" + name, + outs = [out], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % contents, + ) + +def _lua_binary_or_test(name, luamain, luadeps, rule): + script = name + ".sh" + + make_shell_script( + name = "gen_" + name, + out = script, + contents = """ +BASE=$(dirname $(rlocation upb/upb_c.so)) +export LUA_CPATH="$BASE/?.so" +export LUA_PATH="$BASE/?.lua" +$(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" +""", + ) + + rule( + name = name, + srcs = [script], + data = ["@lua//:lua", luamain] + luadeps, + ) + +def lua_binary(name, luamain, luadeps = []): + _lua_binary_or_test(name, luamain, luadeps, native.sh_binary) + +def lua_test(name, luamain, luadeps = []): + _lua_binary_or_test(name, luamain, luadeps, native.sh_test) + +def generated_file_staleness_test(name, outs, generated_pattern): + """Tests that checked-in file(s) match the contents of generated file(s). + + The resulting test will verify that all output files exist and have the + correct contents. If the test fails, it can be invoked with --fix to + bring the checked-in files up to date. + + Args: + name: Name of the rule. + outs: the checked-in files that are copied from generated files. + generated_pattern: the pattern for transforming each "out" file into a + generated file. For example, if generated_pattern="generated/%s" then + a file foo.txt will look for generated file generated/foo.txt. + """ + + script_name = name + ".py" + script_src = "//:tools/staleness_test.py" + + # Filter out non-existing rules so Blaze doesn't error out before we even + # run the test. + existing_outs = native.glob(include = outs) + + # The file list contains a few extra bits of information at the end. + # These get unpacked by the Config class in staleness_test_lib.py. + file_list = outs + [generated_pattern, native.package_name() or ".", name] + + native.genrule( + name = name + "_makescript", + outs = [script_name], + srcs = [script_src], + testonly = 1, + cmd = "cat $(location " + script_src + ") > $@; " + + "sed -i.bak -e 's|INSERT_FILE_LIST_HERE|" + "\\\n ".join(file_list) + "|' $@", + ) + + native.py_test( + name = name, + srcs = [script_name], + data = existing_outs + [generated_pattern % file for file in outs], + deps = [ + "//:staleness_test_lib", + ], + ) + +# upb_amalgamation() rule, with file_list aspect. + +SrcList = provider( + fields = { + "srcs": "list of srcs", + }, +) + +def _file_list_aspect_impl(target, ctx): + if GeneratedSrcsInfo in target: + srcs = target[GeneratedSrcsInfo] + return [SrcList(srcs = srcs.srcs + srcs.hdrs)] + + srcs = [] + for src in ctx.rule.attr.srcs: + srcs += src.files.to_list() + for hdr in ctx.rule.attr.hdrs: + srcs += hdr.files.to_list() + for hdr in ctx.rule.attr.textual_hdrs: + srcs += hdr.files.to_list() + return [SrcList(srcs = srcs)] + +_file_list_aspect = aspect( + implementation = _file_list_aspect_impl, +) + +def _upb_amalgamation(ctx): + inputs = [] + for lib in ctx.attr.libs: + inputs += lib[SrcList].srcs + srcs = [src for src in inputs if src.path.endswith("c")] + ctx.actions.run( + inputs = inputs, + outputs = ctx.outputs.outs, + arguments = [ctx.bin_dir.path + "/"] + [f.path for f in srcs] + ["-I" + root for root in _get_real_roots(inputs)], + progress_message = "Making amalgamation", + executable = ctx.executable.amalgamator, + ) + return [] + +upb_amalgamation = rule( + attrs = { + "amalgamator": attr.label( + executable = True, + cfg = "host", + ), + "libs": attr.label_list(aspects = [_file_list_aspect]), + "outs": attr.output_list(), + }, + implementation = _upb_amalgamation, +) + +def licenses(*args): + # No-op (for Google-internal usage). + pass diff --git a/bazel/lua.BUILD b/bazel/lua.BUILD new file mode 100644 index 00000000000..7be0b594e56 --- /dev/null +++ b/bazel/lua.BUILD @@ -0,0 +1,102 @@ +package( + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "liblua_headers", + defines = ["LUA_USE_LINUX"], + hdrs = [ + "src/lauxlib.h", + "src/lua.h", + "src/lua.hpp", + "src/luaconf.h", + "src/lualib.h", + ], + includes = ["src"], +) + +cc_library( + name = "liblua", + srcs = [ + "src/lapi.c", + "src/lapi.h", + "src/lauxlib.c", + "src/lauxlib.h", + "src/lbaselib.c", + "src/lbitlib.c", + "src/lcode.c", + "src/lcode.h", + "src/lcorolib.c", + "src/lctype.c", + "src/lctype.h", + "src/ldblib.c", + "src/ldebug.c", + "src/ldebug.h", + "src/ldo.c", + "src/ldo.h", + "src/ldump.c", + "src/lfunc.c", + "src/lfunc.h", + "src/lgc.c", + "src/lgc.h", + "src/linit.c", + "src/liolib.c", + "src/llex.c", + "src/llex.h", + "src/llimits.h", + "src/lmathlib.c", + "src/lmem.c", + "src/lmem.h", + "src/loadlib.c", + "src/lobject.c", + "src/lobject.h", + "src/lopcodes.c", + "src/lopcodes.h", + "src/loslib.c", + "src/lparser.c", + "src/lparser.h", + "src/lstate.c", + "src/lstate.h", + "src/lstring.c", + "src/lstring.h", + "src/lstrlib.c", + "src/ltable.c", + "src/ltable.h", + "src/ltablib.c", + "src/ltm.c", + "src/ltm.h", + "src/lundump.c", + "src/lundump.h", + "src/lvm.c", + "src/lvm.h", + "src/lzio.c", + "src/lzio.h", + ], + defines = ["LUA_USE_LINUX"], + hdrs = [ + "src/lauxlib.h", + "src/lua.h", + "src/lua.hpp", + "src/luaconf.h", + "src/lualib.h", + ], + includes = ["src"], + linkopts = [ + "-lm", + "-ldl", + ], +) + +cc_binary( + name = "lua", + srcs = [ + "src/lua.c", + ], + deps = [ + ":liblua", + ], + linkopts = [ + "-lreadline", + "-rdynamic", + ], +) diff --git a/bazel/ragel.BUILD b/bazel/ragel.BUILD new file mode 100644 index 00000000000..5e3b24913df --- /dev/null +++ b/bazel/ragel.BUILD @@ -0,0 +1,193 @@ + +package( + default_visibility = ["//visibility:public"], +) + +cc_binary( + name = "ragelc", + srcs = [ + "ragel/rubycodegen.cpp", + "ragel/goipgoto.h", + "ragel/cdtable.h", + "ragel/rubycodegen.h", + "ragel/gotable.h", + "ragel/gocodegen.cpp", + "ragel/rubyfflat.cpp", + "ragel/common.cpp", + "ragel/gofflat.cpp", + "ragel/cdtable.cpp", + "ragel/cdsplit.cpp", + "ragel/rlparse.cpp", + "ragel/csfgoto.cpp", + "ragel/javacodegen.cpp", + "ragel/gocodegen.h", + "ragel/mlgoto.cpp", + "ragel/fsmgraph.cpp", + "ragel/version.h", + "ragel/mlfflat.h", + "ragel/fsmgraph.h", + "ragel/fsmbase.cpp", + "ragel/fsmstate.cpp", + "ragel/gotablish.cpp", + "ragel/rubyflat.cpp", + "ragel/cdfgoto.h", + "ragel/cscodegen.h", + "ragel/mlflat.cpp", + "ragel/rubyflat.h", + "ragel/goftable.h", + "ragel/rbxgoto.cpp", + "ragel/csfflat.cpp", + "ragel/gofgoto.cpp", + "ragel/gofgoto.h", + "ragel/ragel.h", + "ragel/goftable.cpp", + "ragel/cdcodegen.cpp", + "ragel/rlparse.h", + "ragel/cdsplit.h", + "ragel/xmlcodegen.cpp", + "ragel/goipgoto.cpp", + "ragel/dotcodegen.h", + "ragel/gogoto.cpp", + "ragel/csflat.h", + "ragel/csfflat.h", + #"ragel/config.h.in", + "ragel/csipgoto.cpp", + "ragel/mltable.cpp", + "ragel/mlflat.h", + "ragel/csftable.cpp", + "ragel/cdgoto.h", + "ragel/goflat.cpp", + "ragel/rubyfflat.h", + "ragel/mlftable.h", + "ragel/rubyftable.h", + "ragel/fsmap.cpp", + "ragel/redfsm.cpp", + "ragel/goflat.h", + "ragel/parsetree.cpp", + "ragel/fsmmin.cpp", + "ragel/dotcodegen.cpp", + "ragel/redfsm.h", + "ragel/mlcodegen.cpp", + "ragel/cdfgoto.cpp", + "ragel/cssplit.cpp", + "ragel/cstable.cpp", + "ragel/javacodegen.h", + "ragel/parsedata.cpp", + "ragel/buffer.h", + "ragel/gogoto.h", + "ragel/csgoto.h", + "ragel/pcheck.h", + "ragel/rubyftable.cpp", + "ragel/csfgoto.h", + "ragel/common.h", + "ragel/cdftable.h", + "ragel/mlgoto.h", + "ragel/csgoto.cpp", + "ragel/cdflat.h", + "ragel/cdipgoto.h", + "ragel/cstable.h", + "ragel/gendata.h", + "ragel/cdfflat.cpp", + "ragel/gotable.cpp", + "ragel/cdcodegen.h", + "ragel/gendata.cpp", + "ragel/rubytable.h", + "ragel/csflat.cpp", + "ragel/inputdata.h", + "ragel/inputdata.cpp", + "ragel/rubytable.cpp", + "ragel/fsmattach.cpp", + "ragel/csipgoto.h", + "ragel/cscodegen.cpp", + "ragel/cdfflat.h", + "ragel/rbxgoto.h", + "ragel/xmlcodegen.h", + "ragel/gofflat.h", + "ragel/parsedata.h", + "ragel/mlfgoto.h", + "ragel/cdflat.cpp", + "ragel/config.h", + "ragel/rlscan.cpp", + "ragel/mlcodegen.h", + "ragel/mlfflat.cpp", + "ragel/mlftable.cpp", + "ragel/mltable.h", + "ragel/cdipgoto.cpp", + "ragel/cdftable.cpp", + "ragel/parsetree.h", + "ragel/rlscan.h", + "ragel/main.cpp", + "ragel/cssplit.h", + "ragel/mlfgoto.cpp", + "ragel/csftable.h", + "ragel/gotablish.h", + "ragel/cdgoto.cpp", + "aapl/avlmelkey.h", + "aapl/dlistmel.h", + "aapl/avliset.h", + "aapl/avlkeyless.h", + "aapl/sbstset.h", + "aapl/sbsttable.h", + "aapl/quicksort.h", + "aapl/avlitree.h", + "aapl/avlcommon.h", + "aapl/bstset.h", + "aapl/avlmel.h", + "aapl/insertsort.h", + "aapl/dlist.h", + "aapl/avlmap.h", + "aapl/mergesort.h", + "aapl/resize.h", + "aapl/bstcommon.h", + "aapl/bstmap.h", + "aapl/compare.h", + "aapl/svector.h", + "aapl/avlset.h", + "aapl/bsttable.h", + "aapl/avlikeyless.h", + "aapl/bubblesort.h", + "aapl/table.h", + "aapl/avlbasic.h", + "aapl/vector.h", + "aapl/avlimap.h", + "aapl/dlistval.h", + "aapl/dlcommon.h", + "aapl/avlibasic.h", + "aapl/sbstmap.h", + "aapl/avlimel.h", + "aapl/avlimelkey.h", + "aapl/avltree.h", + ], + includes = ["ragel", "aapl"], +) + +config_h_contents = """ +#define PACKAGE "ragel" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "ragel" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "ragel 6.10" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "ragel" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "6.10" + +/* Version number of package */ +#define VERSION "6.10" +""" + +genrule( + name = "gen_config_h", + outs = ["ragel/config.h"], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % config_h_contents, +) diff --git a/bazel/repository_defs.bzl b/bazel/repository_defs.bzl new file mode 100644 index 00000000000..7b6e78e8b83 --- /dev/null +++ b/bazel/repository_defs.bzl @@ -0,0 +1,15 @@ +# A hacky way to work around the fact that native.bazel_version is only +# available from WORKSPACE macros, not BUILD macros or rules. +# +# Hopefully we can remove this if/when this is fixed: +# https://github.com/bazelbuild/bazel/issues/8305 + +def _impl(repository_ctx): + s = "bazel_version = \"" + native.bazel_version + "\"" + repository_ctx.file("bazel_version.bzl", s) + repository_ctx.file("BUILD", "") + +bazel_version_repository = repository_rule( + implementation = _impl, + local = True, +) diff --git a/bazel/upb_proto_library.bzl b/bazel/upb_proto_library.bzl new file mode 100644 index 00000000000..cc6bcffaede --- /dev/null +++ b/bazel/upb_proto_library.bzl @@ -0,0 +1,299 @@ +"""Public rules for using upb protos: + - upb_proto_library() + - upb_proto_reflection_library() +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +# copybara:strip_for_google3_begin +load("@bazel_skylib//lib:versions.bzl", "versions") +load("@bazel_version//:bazel_version.bzl", "bazel_version") +# copybara:strip_end + +# Generic support code ######################################################### + +_is_bazel = not hasattr(native, "genmpm") + +def _get_real_short_path(file): + # For some reason, files from other archives have short paths that look like: + # ../com_google_protobuf/google/protobuf/descriptor.proto + short_path = file.short_path + if short_path.startswith("../"): + second_slash = short_path.index("/", 3) + short_path = short_path[second_slash + 1:] + return short_path + +def _get_real_root(file): + real_short_path = _get_real_short_path(file) + return file.path[:-len(real_short_path) - 1] + +def _get_real_roots(files): + roots = {} + for file in files: + real_root = _get_real_root(file) + if real_root: + roots[real_root] = True + return roots.keys() + +def _generate_output_file(ctx, src, extension): + real_short_path = _get_real_short_path(src) + real_short_path = paths.relativize(real_short_path, ctx.label.package) + output_filename = paths.replace_extension(real_short_path, extension) + ret = ctx.actions.declare_file(output_filename) + return ret + +def _filter_none(elems): + out = [] + for elem in elems: + if elem: + out.append(elem) + return out + +def _cc_library_func(ctx, name, hdrs, srcs, dep_ccinfos): + """Like cc_library(), but callable from rules. + + Args: + ctx: Rule context. + name: Unique name used to generate output files. + hdrs: Public headers that can be #included from other rules. + srcs: C/C++ source files. + dep_ccinfos: CcInfo providers of dependencies we should build/link against. + + Returns: + CcInfo provider for this compilation. + """ + + compilation_contexts = [info.compilation_context for info in dep_ccinfos] + linking_contexts = [info.linking_context for info in dep_ccinfos] + toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + + # copybara:strip_for_google3_begin + if bazel_version == "0.24.1": + # Compatibility code until gRPC is on 0.25.2 or later. + compilation_info = cc_common.compile( + ctx = ctx, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + srcs = srcs, + hdrs = hdrs, + compilation_contexts = compilation_contexts, + ) + linking_info = cc_common.link( + ctx = ctx, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + cc_compilation_outputs = compilation_info.cc_compilation_outputs, + linking_contexts = linking_contexts, + ) + return CcInfo( + compilation_context = compilation_info.compilation_context, + linking_context = linking_info.linking_context, + ) + + if not versions.is_at_least("0.25.2", bazel_version): + fail("upb requires Bazel >=0.25.2 or 0.24.1") + + # copybara:strip_end + + blaze_only_args = {} + + if not _is_bazel: + blaze_only_args["grep_includes"] = ctx.file._grep_includes + + (compilation_context, compilation_outputs) = cc_common.compile( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + name = name, + srcs = srcs, + public_hdrs = hdrs, + compilation_contexts = compilation_contexts, + **blaze_only_args + ) + (linking_context, linking_outputs) = cc_common.create_linking_context_from_compilation_outputs( + actions = ctx.actions, + name = name, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + compilation_outputs = compilation_outputs, + linking_contexts = linking_contexts, + **blaze_only_args + ) + + return CcInfo( + compilation_context = compilation_context, + linking_context = linking_context, + ) + +# upb_proto_library / upb_proto_reflection_library shared code ################# + +GeneratedSrcsInfo = provider( + fields = { + "srcs": "list of srcs", + "hdrs": "list of hdrs", + }, +) + +_WrappedCcInfo = provider(fields = ["cc_info"]) +_WrappedGeneratedSrcsInfo = provider(fields = ["srcs"]) + +def _compile_upb_protos(ctx, proto_info, proto_sources, ext): + srcs = [_generate_output_file(ctx, name, ext + ".c") for name in proto_sources] + hdrs = [_generate_output_file(ctx, name, ext + ".h") for name in proto_sources] + transitive_sets = proto_info.transitive_descriptor_sets.to_list() + ctx.actions.run( + inputs = depset( + direct = [proto_info.direct_descriptor_set], + transitive = [proto_info.transitive_descriptor_sets], + ), + tools = [ctx.executable._upbc], + outputs = srcs + hdrs, + executable = ctx.executable._protoc, + arguments = [ + "--upb_out=" + _get_real_root(srcs[0]), + "--plugin=protoc-gen-upb=" + ctx.executable._upbc.path, + "--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets]), + ] + + [_get_real_short_path(file) for file in proto_sources], + progress_message = "Generating upb protos for :" + ctx.label.name, + ) + return GeneratedSrcsInfo(srcs = srcs, hdrs = hdrs) + +def _upb_proto_rule_impl(ctx): + if len(ctx.attr.deps) != 1: + fail("only one deps dependency allowed.") + dep = ctx.attr.deps[0] + if _WrappedCcInfo not in dep or _WrappedGeneratedSrcsInfo not in dep: + fail("proto_library rule must generate _WrappedCcInfo and " + + "_WrappedGeneratedSrcsInfo (aspect should have handled this).") + cc_info = dep[_WrappedCcInfo].cc_info + srcs = dep[_WrappedGeneratedSrcsInfo].srcs + if (type(cc_info.linking_context.libraries_to_link) == "list"): + lib = cc_info.linking_context.libraries_to_link[0] + else: + lib = cc_info.linking_context.libraries_to_link.to_list()[0] + files = _filter_none([ + lib.static_library, + lib.pic_static_library, + lib.dynamic_library, + ]) + return [ + DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs)), + srcs, + cc_info, + ] + +def _upb_proto_aspect_impl(target, ctx): + proto_info = target[ProtoInfo] + files = _compile_upb_protos(ctx, proto_info, proto_info.direct_sources, ctx.attr._ext) + deps = ctx.rule.attr.deps + ctx.attr._upb + dep_ccinfos = [dep[CcInfo] for dep in deps if CcInfo in dep] + dep_ccinfos += [dep[_WrappedCcInfo].cc_info for dep in deps if _WrappedCcInfo in dep] + cc_info = _cc_library_func( + ctx = ctx, + name = ctx.rule.attr.name + ctx.attr._ext, + hdrs = files.hdrs, + srcs = files.srcs, + dep_ccinfos = dep_ccinfos, + ) + return [_WrappedCcInfo(cc_info = cc_info), _WrappedGeneratedSrcsInfo(srcs = files)] + +def _maybe_add(d): + if not _is_bazel: + d["_grep_includes"] = attr.label( + allow_single_file = True, + cfg = "host", + default = "//tools/cpp:grep-includes", + ) + return d + +# upb_proto_library() ########################################################## + +_upb_proto_library_aspect = aspect( + attrs = _maybe_add({ + "_upbc": attr.label( + executable = True, + cfg = "host", + default = "//:protoc-gen-upb", + ), + "_protoc": attr.label( + executable = True, + cfg = "host", + default = "@com_google_protobuf//:protoc", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + "_upb": attr.label_list(default = [ + "//:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + "//:upb" + ]), + "_ext": attr.string(default = ".upb"), + }), + implementation = _upb_proto_aspect_impl, + attr_aspects = ["deps"], + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) + +upb_proto_library = rule( + output_to_genfiles = True, + implementation = _upb_proto_rule_impl, + attrs = { + "deps": attr.label_list( + aspects = [_upb_proto_library_aspect], + allow_rules = ["proto_library"], + providers = [ProtoInfo], + ), + }, +) + +# upb_proto_reflection_library() ############################################### + +_upb_proto_reflection_library_aspect = aspect( + attrs = _maybe_add({ + "_upbc": attr.label( + executable = True, + cfg = "host", + default = "//:protoc-gen-upb", + ), + "_protoc": attr.label( + executable = True, + cfg = "host", + default = "@com_google_protobuf//:protoc", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + "_upb": attr.label_list( + default = [ + "//:upb", + "//:reflection", + ], + ), + "_ext": attr.string(default = ".upbdefs"), + }), + implementation = _upb_proto_aspect_impl, + attr_aspects = ["deps"], + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) + +upb_proto_reflection_library = rule( + output_to_genfiles = True, + implementation = _upb_proto_rule_impl, + attrs = { + "deps": attr.label_list( + aspects = [_upb_proto_reflection_library_aspect], + allow_rules = ["proto_library"], + providers = [ProtoInfo], + ), + }, +) diff --git a/bazel/workspace_deps.bzl b/bazel/workspace_deps.bzl new file mode 100644 index 00000000000..39bf524a7a1 --- /dev/null +++ b/bazel/workspace_deps.bzl @@ -0,0 +1,36 @@ + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("//bazel:repository_defs.bzl", "bazel_version_repository") + +def upb_deps(): + bazel_version_repository( + name = "bazel_version", + ) + + git_repository( + name = "absl", + commit = "070f6e47b33a2909d039e620c873204f78809492", + remote = "https://github.com/abseil/abseil-cpp.git", + shallow_since = "1541627663 -0500", + ) + + git_repository( + name = "com_google_protobuf", + remote = "https://github.com/protocolbuffers/protobuf.git", + commit = "d41002663fd04325ead28439dfd5ce2822b0d6fb", + ) + + http_archive( + name = "bazel_skylib", + strip_prefix = "bazel-skylib-master", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/master.tar.gz"], + ) + + http_archive( + name = "zlib", + build_file = "@com_google_protobuf//:third_party/zlib.BUILD", + sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", + strip_prefix = "zlib-1.2.11", + urls = ["https://zlib.net/zlib-1.2.11.tar.gz"], + ) diff --git a/examples/bazel/BUILD b/examples/bazel/BUILD new file mode 100644 index 00000000000..94bcfe64d4f --- /dev/null +++ b/examples/bazel/BUILD @@ -0,0 +1,18 @@ + +load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library") + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) + +upb_proto_library( + name = "foo_upbproto", + deps = [":foo_proto"], +) + +cc_binary( + name = "test_binary", + srcs = ["test_binary.c"], + deps = [":foo_upbproto"], +) diff --git a/examples/bazel/WORKSPACE b/examples/bazel/WORKSPACE new file mode 100644 index 00000000000..5dfe68ac25a --- /dev/null +++ b/examples/bazel/WORKSPACE @@ -0,0 +1,14 @@ + +workspace(name = "upb_example") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "upb", + remote = "https://github.com/protocolbuffers/upb.git", + commit = "d16bf99ac4658793748cda3251226059892b3b7b", +) + +load("@upb//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() diff --git a/examples/bazel/foo.proto b/examples/bazel/foo.proto new file mode 100644 index 00000000000..f68b99443dd --- /dev/null +++ b/examples/bazel/foo.proto @@ -0,0 +1,7 @@ + +syntax = "proto2"; + +message Foo { + optional int64 time = 1; + optional string greeting = 2; +} diff --git a/examples/bazel/test_binary.c b/examples/bazel/test_binary.c new file mode 100644 index 00000000000..78f367a3707 --- /dev/null +++ b/examples/bazel/test_binary.c @@ -0,0 +1,17 @@ + +#include + +#include "foo.upb.h" + +int main() { + upb_arena *arena = upb_arena_new(); + Foo* foo = Foo_new(arena); + const char greeting[] = "Hello, World!\n"; + + Foo_set_time(foo, time(NULL)); + /* Warning: the proto will not copy this, the string data must outlive + * the proto. */ + Foo_set_greeting(foo, upb_strview_makez(greeting)); + + upb_arena_free(arena); +} diff --git a/generated_for_cmake/google/protobuf/descriptor.upb.c b/generated_for_cmake/google/protobuf/descriptor.upb.c new file mode 100644 index 00000000000..61b9299bb43 --- /dev/null +++ b/generated_for_cmake/google/protobuf/descriptor.upb.c @@ -0,0 +1,485 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * google/protobuf/descriptor.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#include +#include "upb/msg.h" +#include "google/protobuf/descriptor.upb.h" + +#include "upb/port_def.inc" + +static const upb_msglayout *const google_protobuf_FileDescriptorSet_submsgs[1] = { + &google_protobuf_FileDescriptorProto_msginit, +}; + +static const upb_msglayout_field google_protobuf_FileDescriptorSet__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_FileDescriptorSet_msginit = { + &google_protobuf_FileDescriptorSet_submsgs[0], + &google_protobuf_FileDescriptorSet__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_protobuf_FileDescriptorProto_submsgs[6] = { + &google_protobuf_DescriptorProto_msginit, + &google_protobuf_EnumDescriptorProto_msginit, + &google_protobuf_FieldDescriptorProto_msginit, + &google_protobuf_FileOptions_msginit, + &google_protobuf_ServiceDescriptorProto_msginit, + &google_protobuf_SourceCodeInfo_msginit, +}; + +static const upb_msglayout_field google_protobuf_FileDescriptorProto__fields[12] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(12, 24), 2, 0, 9, 1}, + {3, UPB_SIZE(36, 72), 0, 0, 9, 3}, + {4, UPB_SIZE(40, 80), 0, 0, 11, 3}, + {5, UPB_SIZE(44, 88), 0, 1, 11, 3}, + {6, UPB_SIZE(48, 96), 0, 4, 11, 3}, + {7, UPB_SIZE(52, 104), 0, 2, 11, 3}, + {8, UPB_SIZE(28, 56), 4, 3, 11, 1}, + {9, UPB_SIZE(32, 64), 5, 5, 11, 1}, + {10, UPB_SIZE(56, 112), 0, 0, 5, 3}, + {11, UPB_SIZE(60, 120), 0, 0, 5, 3}, + {12, UPB_SIZE(20, 40), 3, 0, 9, 1}, +}; + +const upb_msglayout google_protobuf_FileDescriptorProto_msginit = { + &google_protobuf_FileDescriptorProto_submsgs[0], + &google_protobuf_FileDescriptorProto__fields[0], + UPB_SIZE(64, 128), 12, false, +}; + +static const upb_msglayout *const google_protobuf_DescriptorProto_submsgs[8] = { + &google_protobuf_DescriptorProto_msginit, + &google_protobuf_DescriptorProto_ExtensionRange_msginit, + &google_protobuf_DescriptorProto_ReservedRange_msginit, + &google_protobuf_EnumDescriptorProto_msginit, + &google_protobuf_FieldDescriptorProto_msginit, + &google_protobuf_MessageOptions_msginit, + &google_protobuf_OneofDescriptorProto_msginit, +}; + +static const upb_msglayout_field google_protobuf_DescriptorProto__fields[10] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 4, 11, 3}, + {3, UPB_SIZE(20, 40), 0, 0, 11, 3}, + {4, UPB_SIZE(24, 48), 0, 3, 11, 3}, + {5, UPB_SIZE(28, 56), 0, 1, 11, 3}, + {6, UPB_SIZE(32, 64), 0, 4, 11, 3}, + {7, UPB_SIZE(12, 24), 2, 5, 11, 1}, + {8, UPB_SIZE(36, 72), 0, 6, 11, 3}, + {9, UPB_SIZE(40, 80), 0, 2, 11, 3}, + {10, UPB_SIZE(44, 88), 0, 0, 9, 3}, +}; + +const upb_msglayout google_protobuf_DescriptorProto_msginit = { + &google_protobuf_DescriptorProto_submsgs[0], + &google_protobuf_DescriptorProto__fields[0], + UPB_SIZE(48, 96), 10, false, +}; + +static const upb_msglayout *const google_protobuf_DescriptorProto_ExtensionRange_submsgs[1] = { + &google_protobuf_ExtensionRangeOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_DescriptorProto_ExtensionRange__fields[3] = { + {1, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {2, UPB_SIZE(8, 8), 2, 0, 5, 1}, + {3, UPB_SIZE(12, 16), 3, 0, 11, 1}, +}; + +const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit = { + &google_protobuf_DescriptorProto_ExtensionRange_submsgs[0], + &google_protobuf_DescriptorProto_ExtensionRange__fields[0], + UPB_SIZE(16, 24), 3, false, +}; + +static const upb_msglayout_field google_protobuf_DescriptorProto_ReservedRange__fields[2] = { + {1, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {2, UPB_SIZE(8, 8), 2, 0, 5, 1}, +}; + +const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit = { + NULL, + &google_protobuf_DescriptorProto_ReservedRange__fields[0], + UPB_SIZE(12, 12), 2, false, +}; + +static const upb_msglayout *const google_protobuf_ExtensionRangeOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_ExtensionRangeOptions__fields[1] = { + {999, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit = { + &google_protobuf_ExtensionRangeOptions_submsgs[0], + &google_protobuf_ExtensionRangeOptions__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_protobuf_FieldDescriptorProto_submsgs[1] = { + &google_protobuf_FieldOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_FieldDescriptorProto__fields[10] = { + {1, UPB_SIZE(32, 32), 5, 0, 9, 1}, + {2, UPB_SIZE(40, 48), 6, 0, 9, 1}, + {3, UPB_SIZE(24, 24), 3, 0, 5, 1}, + {4, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {5, UPB_SIZE(16, 16), 2, 0, 14, 1}, + {6, UPB_SIZE(48, 64), 7, 0, 9, 1}, + {7, UPB_SIZE(56, 80), 8, 0, 9, 1}, + {8, UPB_SIZE(72, 112), 10, 0, 11, 1}, + {9, UPB_SIZE(28, 28), 4, 0, 5, 1}, + {10, UPB_SIZE(64, 96), 9, 0, 9, 1}, +}; + +const upb_msglayout google_protobuf_FieldDescriptorProto_msginit = { + &google_protobuf_FieldDescriptorProto_submsgs[0], + &google_protobuf_FieldDescriptorProto__fields[0], + UPB_SIZE(80, 128), 10, false, +}; + +static const upb_msglayout *const google_protobuf_OneofDescriptorProto_submsgs[1] = { + &google_protobuf_OneofOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_OneofDescriptorProto__fields[2] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(12, 24), 2, 0, 11, 1}, +}; + +const upb_msglayout google_protobuf_OneofDescriptorProto_msginit = { + &google_protobuf_OneofDescriptorProto_submsgs[0], + &google_protobuf_OneofDescriptorProto__fields[0], + UPB_SIZE(16, 32), 2, false, +}; + +static const upb_msglayout *const google_protobuf_EnumDescriptorProto_submsgs[3] = { + &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, + &google_protobuf_EnumOptions_msginit, + &google_protobuf_EnumValueDescriptorProto_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumDescriptorProto__fields[5] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 2, 11, 3}, + {3, UPB_SIZE(12, 24), 2, 1, 11, 1}, + {4, UPB_SIZE(20, 40), 0, 0, 11, 3}, + {5, UPB_SIZE(24, 48), 0, 0, 9, 3}, +}; + +const upb_msglayout google_protobuf_EnumDescriptorProto_msginit = { + &google_protobuf_EnumDescriptorProto_submsgs[0], + &google_protobuf_EnumDescriptorProto__fields[0], + UPB_SIZE(32, 64), 5, false, +}; + +static const upb_msglayout_field google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[2] = { + {1, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {2, UPB_SIZE(8, 8), 2, 0, 5, 1}, +}; + +const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit = { + NULL, + &google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[0], + UPB_SIZE(12, 12), 2, false, +}; + +static const upb_msglayout *const google_protobuf_EnumValueDescriptorProto_submsgs[1] = { + &google_protobuf_EnumValueOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumValueDescriptorProto__fields[3] = { + {1, UPB_SIZE(8, 8), 2, 0, 9, 1}, + {2, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {3, UPB_SIZE(16, 24), 3, 0, 11, 1}, +}; + +const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit = { + &google_protobuf_EnumValueDescriptorProto_submsgs[0], + &google_protobuf_EnumValueDescriptorProto__fields[0], + UPB_SIZE(24, 32), 3, false, +}; + +static const upb_msglayout *const google_protobuf_ServiceDescriptorProto_submsgs[2] = { + &google_protobuf_MethodDescriptorProto_msginit, + &google_protobuf_ServiceOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_ServiceDescriptorProto__fields[3] = { + {1, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 0, 11, 3}, + {3, UPB_SIZE(12, 24), 2, 1, 11, 1}, +}; + +const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit = { + &google_protobuf_ServiceDescriptorProto_submsgs[0], + &google_protobuf_ServiceDescriptorProto__fields[0], + UPB_SIZE(24, 48), 3, false, +}; + +static const upb_msglayout *const google_protobuf_MethodDescriptorProto_submsgs[1] = { + &google_protobuf_MethodOptions_msginit, +}; + +static const upb_msglayout_field google_protobuf_MethodDescriptorProto__fields[6] = { + {1, UPB_SIZE(4, 8), 3, 0, 9, 1}, + {2, UPB_SIZE(12, 24), 4, 0, 9, 1}, + {3, UPB_SIZE(20, 40), 5, 0, 9, 1}, + {4, UPB_SIZE(28, 56), 6, 0, 11, 1}, + {5, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {6, UPB_SIZE(2, 2), 2, 0, 8, 1}, +}; + +const upb_msglayout google_protobuf_MethodDescriptorProto_msginit = { + &google_protobuf_MethodDescriptorProto_submsgs[0], + &google_protobuf_MethodDescriptorProto__fields[0], + UPB_SIZE(32, 64), 6, false, +}; + +static const upb_msglayout *const google_protobuf_FileOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_FileOptions__fields[21] = { + {1, UPB_SIZE(28, 32), 11, 0, 9, 1}, + {8, UPB_SIZE(36, 48), 12, 0, 9, 1}, + {9, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {10, UPB_SIZE(16, 16), 2, 0, 8, 1}, + {11, UPB_SIZE(44, 64), 13, 0, 9, 1}, + {16, UPB_SIZE(17, 17), 3, 0, 8, 1}, + {17, UPB_SIZE(18, 18), 4, 0, 8, 1}, + {18, UPB_SIZE(19, 19), 5, 0, 8, 1}, + {20, UPB_SIZE(20, 20), 6, 0, 8, 1}, + {23, UPB_SIZE(21, 21), 7, 0, 8, 1}, + {27, UPB_SIZE(22, 22), 8, 0, 8, 1}, + {31, UPB_SIZE(23, 23), 9, 0, 8, 1}, + {36, UPB_SIZE(52, 80), 14, 0, 9, 1}, + {37, UPB_SIZE(60, 96), 15, 0, 9, 1}, + {39, UPB_SIZE(68, 112), 16, 0, 9, 1}, + {40, UPB_SIZE(76, 128), 17, 0, 9, 1}, + {41, UPB_SIZE(84, 144), 18, 0, 9, 1}, + {42, UPB_SIZE(24, 24), 10, 0, 8, 1}, + {44, UPB_SIZE(92, 160), 19, 0, 9, 1}, + {45, UPB_SIZE(100, 176), 20, 0, 9, 1}, + {999, UPB_SIZE(108, 192), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_FileOptions_msginit = { + &google_protobuf_FileOptions_submsgs[0], + &google_protobuf_FileOptions__fields[0], + UPB_SIZE(112, 208), 21, false, +}; + +static const upb_msglayout *const google_protobuf_MessageOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_MessageOptions__fields[5] = { + {1, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {2, UPB_SIZE(2, 2), 2, 0, 8, 1}, + {3, UPB_SIZE(3, 3), 3, 0, 8, 1}, + {7, UPB_SIZE(4, 4), 4, 0, 8, 1}, + {999, UPB_SIZE(8, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_MessageOptions_msginit = { + &google_protobuf_MessageOptions_submsgs[0], + &google_protobuf_MessageOptions__fields[0], + UPB_SIZE(12, 16), 5, false, +}; + +static const upb_msglayout *const google_protobuf_FieldOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_FieldOptions__fields[7] = { + {1, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {2, UPB_SIZE(24, 24), 3, 0, 8, 1}, + {3, UPB_SIZE(25, 25), 4, 0, 8, 1}, + {5, UPB_SIZE(26, 26), 5, 0, 8, 1}, + {6, UPB_SIZE(16, 16), 2, 0, 14, 1}, + {10, UPB_SIZE(27, 27), 6, 0, 8, 1}, + {999, UPB_SIZE(28, 32), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_FieldOptions_msginit = { + &google_protobuf_FieldOptions_submsgs[0], + &google_protobuf_FieldOptions__fields[0], + UPB_SIZE(32, 40), 7, false, +}; + +static const upb_msglayout *const google_protobuf_OneofOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_OneofOptions__fields[1] = { + {999, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_OneofOptions_msginit = { + &google_protobuf_OneofOptions_submsgs[0], + &google_protobuf_OneofOptions__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_protobuf_EnumOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumOptions__fields[3] = { + {2, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {3, UPB_SIZE(2, 2), 2, 0, 8, 1}, + {999, UPB_SIZE(4, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_EnumOptions_msginit = { + &google_protobuf_EnumOptions_submsgs[0], + &google_protobuf_EnumOptions__fields[0], + UPB_SIZE(8, 16), 3, false, +}; + +static const upb_msglayout *const google_protobuf_EnumValueOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_EnumValueOptions__fields[2] = { + {1, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {999, UPB_SIZE(4, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_EnumValueOptions_msginit = { + &google_protobuf_EnumValueOptions_submsgs[0], + &google_protobuf_EnumValueOptions__fields[0], + UPB_SIZE(8, 16), 2, false, +}; + +static const upb_msglayout *const google_protobuf_ServiceOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_ServiceOptions__fields[2] = { + {33, UPB_SIZE(1, 1), 1, 0, 8, 1}, + {999, UPB_SIZE(4, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_ServiceOptions_msginit = { + &google_protobuf_ServiceOptions_submsgs[0], + &google_protobuf_ServiceOptions__fields[0], + UPB_SIZE(8, 16), 2, false, +}; + +static const upb_msglayout *const google_protobuf_MethodOptions_submsgs[1] = { + &google_protobuf_UninterpretedOption_msginit, +}; + +static const upb_msglayout_field google_protobuf_MethodOptions__fields[3] = { + {33, UPB_SIZE(16, 16), 2, 0, 8, 1}, + {34, UPB_SIZE(8, 8), 1, 0, 14, 1}, + {999, UPB_SIZE(20, 24), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_MethodOptions_msginit = { + &google_protobuf_MethodOptions_submsgs[0], + &google_protobuf_MethodOptions__fields[0], + UPB_SIZE(24, 32), 3, false, +}; + +static const upb_msglayout *const google_protobuf_UninterpretedOption_submsgs[1] = { + &google_protobuf_UninterpretedOption_NamePart_msginit, +}; + +static const upb_msglayout_field google_protobuf_UninterpretedOption__fields[7] = { + {2, UPB_SIZE(56, 80), 0, 0, 11, 3}, + {3, UPB_SIZE(32, 32), 4, 0, 9, 1}, + {4, UPB_SIZE(8, 8), 1, 0, 4, 1}, + {5, UPB_SIZE(16, 16), 2, 0, 3, 1}, + {6, UPB_SIZE(24, 24), 3, 0, 1, 1}, + {7, UPB_SIZE(40, 48), 5, 0, 12, 1}, + {8, UPB_SIZE(48, 64), 6, 0, 9, 1}, +}; + +const upb_msglayout google_protobuf_UninterpretedOption_msginit = { + &google_protobuf_UninterpretedOption_submsgs[0], + &google_protobuf_UninterpretedOption__fields[0], + UPB_SIZE(64, 96), 7, false, +}; + +static const upb_msglayout_field google_protobuf_UninterpretedOption_NamePart__fields[2] = { + {1, UPB_SIZE(4, 8), 2, 0, 9, 2}, + {2, UPB_SIZE(1, 1), 1, 0, 8, 2}, +}; + +const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit = { + NULL, + &google_protobuf_UninterpretedOption_NamePart__fields[0], + UPB_SIZE(16, 32), 2, false, +}; + +static const upb_msglayout *const google_protobuf_SourceCodeInfo_submsgs[1] = { + &google_protobuf_SourceCodeInfo_Location_msginit, +}; + +static const upb_msglayout_field google_protobuf_SourceCodeInfo__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_SourceCodeInfo_msginit = { + &google_protobuf_SourceCodeInfo_submsgs[0], + &google_protobuf_SourceCodeInfo__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout_field google_protobuf_SourceCodeInfo_Location__fields[5] = { + {1, UPB_SIZE(20, 40), 0, 0, 5, 3}, + {2, UPB_SIZE(24, 48), 0, 0, 5, 3}, + {3, UPB_SIZE(4, 8), 1, 0, 9, 1}, + {4, UPB_SIZE(12, 24), 2, 0, 9, 1}, + {6, UPB_SIZE(28, 56), 0, 0, 9, 3}, +}; + +const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit = { + NULL, + &google_protobuf_SourceCodeInfo_Location__fields[0], + UPB_SIZE(32, 64), 5, false, +}; + +static const upb_msglayout *const google_protobuf_GeneratedCodeInfo_submsgs[1] = { + &google_protobuf_GeneratedCodeInfo_Annotation_msginit, +}; + +static const upb_msglayout_field google_protobuf_GeneratedCodeInfo__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit = { + &google_protobuf_GeneratedCodeInfo_submsgs[0], + &google_protobuf_GeneratedCodeInfo__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout_field google_protobuf_GeneratedCodeInfo_Annotation__fields[4] = { + {1, UPB_SIZE(20, 32), 0, 0, 5, 3}, + {2, UPB_SIZE(12, 16), 3, 0, 9, 1}, + {3, UPB_SIZE(4, 4), 1, 0, 5, 1}, + {4, UPB_SIZE(8, 8), 2, 0, 5, 1}, +}; + +const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit = { + NULL, + &google_protobuf_GeneratedCodeInfo_Annotation__fields[0], + UPB_SIZE(24, 48), 4, false, +}; + +#include "upb/port_undef.inc" + diff --git a/generated_for_cmake/google/protobuf/descriptor.upb.h b/generated_for_cmake/google/protobuf/descriptor.upb.h new file mode 100644 index 00000000000..681614910e0 --- /dev/null +++ b/generated_for_cmake/google/protobuf/descriptor.upb.h @@ -0,0 +1,1690 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * google/protobuf/descriptor.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ +#define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ + +#include "upb/generated_util.h" +#include "upb/msg.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +struct google_protobuf_FileDescriptorSet; +struct google_protobuf_FileDescriptorProto; +struct google_protobuf_DescriptorProto; +struct google_protobuf_DescriptorProto_ExtensionRange; +struct google_protobuf_DescriptorProto_ReservedRange; +struct google_protobuf_ExtensionRangeOptions; +struct google_protobuf_FieldDescriptorProto; +struct google_protobuf_OneofDescriptorProto; +struct google_protobuf_EnumDescriptorProto; +struct google_protobuf_EnumDescriptorProto_EnumReservedRange; +struct google_protobuf_EnumValueDescriptorProto; +struct google_protobuf_ServiceDescriptorProto; +struct google_protobuf_MethodDescriptorProto; +struct google_protobuf_FileOptions; +struct google_protobuf_MessageOptions; +struct google_protobuf_FieldOptions; +struct google_protobuf_OneofOptions; +struct google_protobuf_EnumOptions; +struct google_protobuf_EnumValueOptions; +struct google_protobuf_ServiceOptions; +struct google_protobuf_MethodOptions; +struct google_protobuf_UninterpretedOption; +struct google_protobuf_UninterpretedOption_NamePart; +struct google_protobuf_SourceCodeInfo; +struct google_protobuf_SourceCodeInfo_Location; +struct google_protobuf_GeneratedCodeInfo; +struct google_protobuf_GeneratedCodeInfo_Annotation; +typedef struct google_protobuf_FileDescriptorSet google_protobuf_FileDescriptorSet; +typedef struct google_protobuf_FileDescriptorProto google_protobuf_FileDescriptorProto; +typedef struct google_protobuf_DescriptorProto google_protobuf_DescriptorProto; +typedef struct google_protobuf_DescriptorProto_ExtensionRange google_protobuf_DescriptorProto_ExtensionRange; +typedef struct google_protobuf_DescriptorProto_ReservedRange google_protobuf_DescriptorProto_ReservedRange; +typedef struct google_protobuf_ExtensionRangeOptions google_protobuf_ExtensionRangeOptions; +typedef struct google_protobuf_FieldDescriptorProto google_protobuf_FieldDescriptorProto; +typedef struct google_protobuf_OneofDescriptorProto google_protobuf_OneofDescriptorProto; +typedef struct google_protobuf_EnumDescriptorProto google_protobuf_EnumDescriptorProto; +typedef struct google_protobuf_EnumDescriptorProto_EnumReservedRange google_protobuf_EnumDescriptorProto_EnumReservedRange; +typedef struct google_protobuf_EnumValueDescriptorProto google_protobuf_EnumValueDescriptorProto; +typedef struct google_protobuf_ServiceDescriptorProto google_protobuf_ServiceDescriptorProto; +typedef struct google_protobuf_MethodDescriptorProto google_protobuf_MethodDescriptorProto; +typedef struct google_protobuf_FileOptions google_protobuf_FileOptions; +typedef struct google_protobuf_MessageOptions google_protobuf_MessageOptions; +typedef struct google_protobuf_FieldOptions google_protobuf_FieldOptions; +typedef struct google_protobuf_OneofOptions google_protobuf_OneofOptions; +typedef struct google_protobuf_EnumOptions google_protobuf_EnumOptions; +typedef struct google_protobuf_EnumValueOptions google_protobuf_EnumValueOptions; +typedef struct google_protobuf_ServiceOptions google_protobuf_ServiceOptions; +typedef struct google_protobuf_MethodOptions google_protobuf_MethodOptions; +typedef struct google_protobuf_UninterpretedOption google_protobuf_UninterpretedOption; +typedef struct google_protobuf_UninterpretedOption_NamePart google_protobuf_UninterpretedOption_NamePart; +typedef struct google_protobuf_SourceCodeInfo google_protobuf_SourceCodeInfo; +typedef struct google_protobuf_SourceCodeInfo_Location google_protobuf_SourceCodeInfo_Location; +typedef struct google_protobuf_GeneratedCodeInfo google_protobuf_GeneratedCodeInfo; +typedef struct google_protobuf_GeneratedCodeInfo_Annotation google_protobuf_GeneratedCodeInfo_Annotation; +extern const upb_msglayout google_protobuf_FileDescriptorSet_msginit; +extern const upb_msglayout google_protobuf_FileDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_DescriptorProto_msginit; +extern const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit; +extern const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit; +extern const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit; +extern const upb_msglayout google_protobuf_FieldDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_OneofDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_EnumDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit; +extern const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_MethodDescriptorProto_msginit; +extern const upb_msglayout google_protobuf_FileOptions_msginit; +extern const upb_msglayout google_protobuf_MessageOptions_msginit; +extern const upb_msglayout google_protobuf_FieldOptions_msginit; +extern const upb_msglayout google_protobuf_OneofOptions_msginit; +extern const upb_msglayout google_protobuf_EnumOptions_msginit; +extern const upb_msglayout google_protobuf_EnumValueOptions_msginit; +extern const upb_msglayout google_protobuf_ServiceOptions_msginit; +extern const upb_msglayout google_protobuf_MethodOptions_msginit; +extern const upb_msglayout google_protobuf_UninterpretedOption_msginit; +extern const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit; +extern const upb_msglayout google_protobuf_SourceCodeInfo_msginit; +extern const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit; +extern const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit; +extern const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit; + +typedef enum { + google_protobuf_FieldDescriptorProto_LABEL_OPTIONAL = 1, + google_protobuf_FieldDescriptorProto_LABEL_REQUIRED = 2, + google_protobuf_FieldDescriptorProto_LABEL_REPEATED = 3 +} google_protobuf_FieldDescriptorProto_Label; + +typedef enum { + google_protobuf_FieldDescriptorProto_TYPE_DOUBLE = 1, + google_protobuf_FieldDescriptorProto_TYPE_FLOAT = 2, + google_protobuf_FieldDescriptorProto_TYPE_INT64 = 3, + google_protobuf_FieldDescriptorProto_TYPE_UINT64 = 4, + google_protobuf_FieldDescriptorProto_TYPE_INT32 = 5, + google_protobuf_FieldDescriptorProto_TYPE_FIXED64 = 6, + google_protobuf_FieldDescriptorProto_TYPE_FIXED32 = 7, + google_protobuf_FieldDescriptorProto_TYPE_BOOL = 8, + google_protobuf_FieldDescriptorProto_TYPE_STRING = 9, + google_protobuf_FieldDescriptorProto_TYPE_GROUP = 10, + google_protobuf_FieldDescriptorProto_TYPE_MESSAGE = 11, + google_protobuf_FieldDescriptorProto_TYPE_BYTES = 12, + google_protobuf_FieldDescriptorProto_TYPE_UINT32 = 13, + google_protobuf_FieldDescriptorProto_TYPE_ENUM = 14, + google_protobuf_FieldDescriptorProto_TYPE_SFIXED32 = 15, + google_protobuf_FieldDescriptorProto_TYPE_SFIXED64 = 16, + google_protobuf_FieldDescriptorProto_TYPE_SINT32 = 17, + google_protobuf_FieldDescriptorProto_TYPE_SINT64 = 18 +} google_protobuf_FieldDescriptorProto_Type; + +typedef enum { + google_protobuf_FieldOptions_STRING = 0, + google_protobuf_FieldOptions_CORD = 1, + google_protobuf_FieldOptions_STRING_PIECE = 2 +} google_protobuf_FieldOptions_CType; + +typedef enum { + google_protobuf_FieldOptions_JS_NORMAL = 0, + google_protobuf_FieldOptions_JS_STRING = 1, + google_protobuf_FieldOptions_JS_NUMBER = 2 +} google_protobuf_FieldOptions_JSType; + +typedef enum { + google_protobuf_FileOptions_SPEED = 1, + google_protobuf_FileOptions_CODE_SIZE = 2, + google_protobuf_FileOptions_LITE_RUNTIME = 3 +} google_protobuf_FileOptions_OptimizeMode; + +typedef enum { + google_protobuf_MethodOptions_IDEMPOTENCY_UNKNOWN = 0, + google_protobuf_MethodOptions_NO_SIDE_EFFECTS = 1, + google_protobuf_MethodOptions_IDEMPOTENT = 2 +} google_protobuf_MethodOptions_IdempotencyLevel; + + +/* google.protobuf.FileDescriptorSet */ + +UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_new(upb_arena *arena) { + return (google_protobuf_FileDescriptorSet *)upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena); +} +UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FileDescriptorSet *ret = google_protobuf_FileDescriptorSet_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FileDescriptorSet_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FileDescriptorSet_serialize(const google_protobuf_FileDescriptorSet *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FileDescriptorSet_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_FileDescriptorProto* const* google_protobuf_FileDescriptorSet_file(const google_protobuf_FileDescriptorSet *msg, size_t *len) { return (const google_protobuf_FileDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_FileDescriptorProto** google_protobuf_FileDescriptorSet_mutable_file(google_protobuf_FileDescriptorSet *msg, size_t *len) { + return (google_protobuf_FileDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_FileDescriptorProto** google_protobuf_FileDescriptorSet_resize_file(google_protobuf_FileDescriptorSet *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FileDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FileDescriptorProto* google_protobuf_FileDescriptorSet_add_file(google_protobuf_FileDescriptorSet *msg, upb_arena *arena) { + struct google_protobuf_FileDescriptorProto* sub = (struct google_protobuf_FileDescriptorProto*)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.FileDescriptorProto */ + +UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_FileDescriptorProto *)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FileDescriptorProto *ret = google_protobuf_FileDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FileDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FileDescriptorProto_serialize(const google_protobuf_FileDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FileDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_name(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_FileDescriptorProto_name(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_package(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_FileDescriptorProto_package(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)); } +UPB_INLINE upb_strview const* google_protobuf_FileDescriptorProto_dependency(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(36, 72), len); } +UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_FileDescriptorProto_message_type(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(40, 80), len); } +UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_FileDescriptorProto_enum_type(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(44, 88), len); } +UPB_INLINE const google_protobuf_ServiceDescriptorProto* const* google_protobuf_FileDescriptorProto_service(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_ServiceDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(48, 96), len); } +UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_FileDescriptorProto_extension(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (const google_protobuf_FieldDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(52, 104), len); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_options(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE const google_protobuf_FileOptions* google_protobuf_FileDescriptorProto_options(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_FileOptions*, UPB_SIZE(28, 56)); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_source_code_info(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE const google_protobuf_SourceCodeInfo* google_protobuf_FileDescriptorProto_source_code_info(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_SourceCodeInfo*, UPB_SIZE(32, 64)); } +UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_public_dependency(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(56, 112), len); } +UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_weak_dependency(const google_protobuf_FileDescriptorProto *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(60, 120), len); } +UPB_INLINE bool google_protobuf_FileDescriptorProto_has_syntax(const google_protobuf_FileDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE upb_strview google_protobuf_FileDescriptorProto_syntax(const google_protobuf_FileDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)); } + +UPB_INLINE void google_protobuf_FileDescriptorProto_set_name(google_protobuf_FileDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_package(google_protobuf_FileDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE upb_strview* google_protobuf_FileDescriptorProto_mutable_dependency(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(36, 72), len); +} +UPB_INLINE upb_strview* google_protobuf_FileDescriptorProto_resize_dependency(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(36, 72), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_FileDescriptorProto_add_dependency(google_protobuf_FileDescriptorProto *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(36, 72), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_FileDescriptorProto_mutable_message_type(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(40, 80), len); +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_FileDescriptorProto_resize_message_type(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(40, 80), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto* google_protobuf_FileDescriptorProto_add_message_type(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto* sub = (struct google_protobuf_DescriptorProto*)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(40, 80), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_FileDescriptorProto_mutable_enum_type(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(44, 88), len); +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_FileDescriptorProto_resize_enum_type(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(44, 88), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumDescriptorProto* google_protobuf_FileDescriptorProto_add_enum_type(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumDescriptorProto* sub = (struct google_protobuf_EnumDescriptorProto*)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(44, 88), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_ServiceDescriptorProto** google_protobuf_FileDescriptorProto_mutable_service(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_ServiceDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(48, 96), len); +} +UPB_INLINE google_protobuf_ServiceDescriptorProto** google_protobuf_FileDescriptorProto_resize_service(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_ServiceDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(48, 96), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_ServiceDescriptorProto* google_protobuf_FileDescriptorProto_add_service(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_ServiceDescriptorProto* sub = (struct google_protobuf_ServiceDescriptorProto*)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(48, 96), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_FileDescriptorProto_mutable_extension(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(52, 104), len); +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_FileDescriptorProto_resize_extension(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(52, 104), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FieldDescriptorProto* google_protobuf_FileDescriptorProto_add_extension(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldDescriptorProto* sub = (struct google_protobuf_FieldDescriptorProto*)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(52, 104), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_options(google_protobuf_FileDescriptorProto *msg, google_protobuf_FileOptions* value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, google_protobuf_FileOptions*, UPB_SIZE(28, 56)) = value; +} +UPB_INLINE struct google_protobuf_FileOptions* google_protobuf_FileDescriptorProto_mutable_options(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FileOptions* sub = (struct google_protobuf_FileOptions*)google_protobuf_FileDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_FileOptions*)upb_msg_new(&google_protobuf_FileOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_FileDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_source_code_info(google_protobuf_FileDescriptorProto *msg, google_protobuf_SourceCodeInfo* value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, google_protobuf_SourceCodeInfo*, UPB_SIZE(32, 64)) = value; +} +UPB_INLINE struct google_protobuf_SourceCodeInfo* google_protobuf_FileDescriptorProto_mutable_source_code_info(google_protobuf_FileDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_SourceCodeInfo* sub = (struct google_protobuf_SourceCodeInfo*)google_protobuf_FileDescriptorProto_source_code_info(msg); + if (sub == NULL) { + sub = (struct google_protobuf_SourceCodeInfo*)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); + if (!sub) return NULL; + google_protobuf_FileDescriptorProto_set_source_code_info(msg, sub); + } + return sub; +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_mutable_public_dependency(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(56, 112), len); +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_resize_public_dependency(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(56, 112), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_FileDescriptorProto_add_public_dependency(google_protobuf_FileDescriptorProto *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(56, 112), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_mutable_weak_dependency(google_protobuf_FileDescriptorProto *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(60, 120), len); +} +UPB_INLINE int32_t* google_protobuf_FileDescriptorProto_resize_weak_dependency(google_protobuf_FileDescriptorProto *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(60, 120), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_FileDescriptorProto_add_weak_dependency(google_protobuf_FileDescriptorProto *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(60, 120), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE void google_protobuf_FileDescriptorProto_set_syntax(google_protobuf_FileDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)) = value; +} + +/* google.protobuf.DescriptorProto */ + +UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_new(upb_arena *arena) { + return (google_protobuf_DescriptorProto *)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_DescriptorProto *ret = google_protobuf_DescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_DescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_DescriptorProto_serialize(const google_protobuf_DescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_DescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_DescriptorProto_has_name(const google_protobuf_DescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_DescriptorProto_name(const google_protobuf_DescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_DescriptorProto_field(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_FieldDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_DescriptorProto_nested_type(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } +UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_DescriptorProto_enum_type(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(24, 48), len); } +UPB_INLINE const google_protobuf_DescriptorProto_ExtensionRange* const* google_protobuf_DescriptorProto_extension_range(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto_ExtensionRange* const*)_upb_array_accessor(msg, UPB_SIZE(28, 56), len); } +UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_DescriptorProto_extension(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_FieldDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(32, 64), len); } +UPB_INLINE bool google_protobuf_DescriptorProto_has_options(const google_protobuf_DescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_MessageOptions* google_protobuf_DescriptorProto_options(const google_protobuf_DescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_MessageOptions*, UPB_SIZE(12, 24)); } +UPB_INLINE const google_protobuf_OneofDescriptorProto* const* google_protobuf_DescriptorProto_oneof_decl(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_OneofDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(36, 72), len); } +UPB_INLINE const google_protobuf_DescriptorProto_ReservedRange* const* google_protobuf_DescriptorProto_reserved_range(const google_protobuf_DescriptorProto *msg, size_t *len) { return (const google_protobuf_DescriptorProto_ReservedRange* const*)_upb_array_accessor(msg, UPB_SIZE(40, 80), len); } +UPB_INLINE upb_strview const* google_protobuf_DescriptorProto_reserved_name(const google_protobuf_DescriptorProto *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(44, 88), len); } + +UPB_INLINE void google_protobuf_DescriptorProto_set_name(google_protobuf_DescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_mutable_field(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_resize_field(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FieldDescriptorProto* google_protobuf_DescriptorProto_add_field(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldDescriptorProto* sub = (struct google_protobuf_FieldDescriptorProto*)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_DescriptorProto_mutable_nested_type(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE google_protobuf_DescriptorProto** google_protobuf_DescriptorProto_resize_nested_type(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto* google_protobuf_DescriptorProto_add_nested_type(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto* sub = (struct google_protobuf_DescriptorProto*)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_DescriptorProto_mutable_enum_type(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(24, 48), len); +} +UPB_INLINE google_protobuf_EnumDescriptorProto** google_protobuf_DescriptorProto_resize_enum_type(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(24, 48), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumDescriptorProto* google_protobuf_DescriptorProto_add_enum_type(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumDescriptorProto* sub = (struct google_protobuf_EnumDescriptorProto*)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(24, 48), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange** google_protobuf_DescriptorProto_mutable_extension_range(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto_ExtensionRange**)_upb_array_mutable_accessor(msg, UPB_SIZE(28, 56), len); +} +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange** google_protobuf_DescriptorProto_resize_extension_range(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto_ExtensionRange**)_upb_array_resize_accessor(msg, UPB_SIZE(28, 56), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto_ExtensionRange* google_protobuf_DescriptorProto_add_extension_range(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto_ExtensionRange* sub = (struct google_protobuf_DescriptorProto_ExtensionRange*)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(28, 56), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_mutable_extension(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(32, 64), len); +} +UPB_INLINE google_protobuf_FieldDescriptorProto** google_protobuf_DescriptorProto_resize_extension(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(32, 64), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_FieldDescriptorProto* google_protobuf_DescriptorProto_add_extension(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldDescriptorProto* sub = (struct google_protobuf_FieldDescriptorProto*)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(32, 64), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_DescriptorProto_set_options(google_protobuf_DescriptorProto *msg, google_protobuf_MessageOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_MessageOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_MessageOptions* google_protobuf_DescriptorProto_mutable_options(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_MessageOptions* sub = (struct google_protobuf_MessageOptions*)google_protobuf_DescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_MessageOptions*)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_DescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE google_protobuf_OneofDescriptorProto** google_protobuf_DescriptorProto_mutable_oneof_decl(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_OneofDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(36, 72), len); +} +UPB_INLINE google_protobuf_OneofDescriptorProto** google_protobuf_DescriptorProto_resize_oneof_decl(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_OneofDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(36, 72), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_OneofDescriptorProto* google_protobuf_DescriptorProto_add_oneof_decl(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_OneofDescriptorProto* sub = (struct google_protobuf_OneofDescriptorProto*)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(36, 72), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange** google_protobuf_DescriptorProto_mutable_reserved_range(google_protobuf_DescriptorProto *msg, size_t *len) { + return (google_protobuf_DescriptorProto_ReservedRange**)_upb_array_mutable_accessor(msg, UPB_SIZE(40, 80), len); +} +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange** google_protobuf_DescriptorProto_resize_reserved_range(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_DescriptorProto_ReservedRange**)_upb_array_resize_accessor(msg, UPB_SIZE(40, 80), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_DescriptorProto_ReservedRange* google_protobuf_DescriptorProto_add_reserved_range(google_protobuf_DescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_DescriptorProto_ReservedRange* sub = (struct google_protobuf_DescriptorProto_ReservedRange*)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(40, 80), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE upb_strview* google_protobuf_DescriptorProto_mutable_reserved_name(google_protobuf_DescriptorProto *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(44, 88), len); +} +UPB_INLINE upb_strview* google_protobuf_DescriptorProto_resize_reserved_name(google_protobuf_DescriptorProto *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(44, 88), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_DescriptorProto_add_reserved_name(google_protobuf_DescriptorProto *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(44, 88), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} + +/* google.protobuf.DescriptorProto.ExtensionRange */ + +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_new(upb_arena *arena) { + return (google_protobuf_DescriptorProto_ExtensionRange *)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); +} +UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_DescriptorProto_ExtensionRange *ret = google_protobuf_DescriptorProto_ExtensionRange_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_DescriptorProto_ExtensionRange_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_DescriptorProto_ExtensionRange_serialize(const google_protobuf_DescriptorProto_ExtensionRange *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_DescriptorProto_ExtensionRange_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_DescriptorProto_ExtensionRange_has_start(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ExtensionRange_start(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_DescriptorProto_ExtensionRange_has_end(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ExtensionRange_end(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_DescriptorProto_ExtensionRange_has_options(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE const google_protobuf_ExtensionRangeOptions* google_protobuf_DescriptorProto_ExtensionRange_options(const google_protobuf_DescriptorProto_ExtensionRange *msg) { return UPB_FIELD_AT(msg, const google_protobuf_ExtensionRangeOptions*, UPB_SIZE(12, 16)); } + +UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_start(google_protobuf_DescriptorProto_ExtensionRange *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_end(google_protobuf_DescriptorProto_ExtensionRange *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_options(google_protobuf_DescriptorProto_ExtensionRange *msg, google_protobuf_ExtensionRangeOptions* value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, google_protobuf_ExtensionRangeOptions*, UPB_SIZE(12, 16)) = value; +} +UPB_INLINE struct google_protobuf_ExtensionRangeOptions* google_protobuf_DescriptorProto_ExtensionRange_mutable_options(google_protobuf_DescriptorProto_ExtensionRange *msg, upb_arena *arena) { + struct google_protobuf_ExtensionRangeOptions* sub = (struct google_protobuf_ExtensionRangeOptions*)google_protobuf_DescriptorProto_ExtensionRange_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_ExtensionRangeOptions*)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_DescriptorProto_ExtensionRange_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.DescriptorProto.ReservedRange */ + +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_new(upb_arena *arena) { + return (google_protobuf_DescriptorProto_ReservedRange *)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); +} +UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_DescriptorProto_ReservedRange *ret = google_protobuf_DescriptorProto_ReservedRange_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_DescriptorProto_ReservedRange_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_DescriptorProto_ReservedRange_serialize(const google_protobuf_DescriptorProto_ReservedRange *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_DescriptorProto_ReservedRange_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_DescriptorProto_ReservedRange_has_start(const google_protobuf_DescriptorProto_ReservedRange *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ReservedRange_start(const google_protobuf_DescriptorProto_ReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_DescriptorProto_ReservedRange_has_end(const google_protobuf_DescriptorProto_ReservedRange *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_DescriptorProto_ReservedRange_end(const google_protobuf_DescriptorProto_ReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_start(google_protobuf_DescriptorProto_ReservedRange *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_end(google_protobuf_DescriptorProto_ReservedRange *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +/* google.protobuf.ExtensionRangeOptions */ + +UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_new(upb_arena *arena) { + return (google_protobuf_ExtensionRangeOptions *)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); +} +UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_ExtensionRangeOptions *ret = google_protobuf_ExtensionRangeOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_ExtensionRangeOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_ExtensionRangeOptions_serialize(const google_protobuf_ExtensionRangeOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_ExtensionRangeOptions_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_ExtensionRangeOptions_uninterpreted_option(const google_protobuf_ExtensionRangeOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ExtensionRangeOptions_mutable_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ExtensionRangeOptions_resize_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_ExtensionRangeOptions_add_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.FieldDescriptorProto */ + +UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_FieldDescriptorProto *)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FieldDescriptorProto *ret = google_protobuf_FieldDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FieldDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FieldDescriptorProto_serialize(const google_protobuf_FieldDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FieldDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_name(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_name(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_extendee(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_extendee(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_number(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_number(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_label(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_label(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_type(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_type(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_type_name(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 7); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_type_name(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_default_value(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 8); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_default_value(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(56, 80)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_options(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 10); } +UPB_INLINE const google_protobuf_FieldOptions* google_protobuf_FieldDescriptorProto_options(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_FieldOptions*, UPB_SIZE(72, 112)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_oneof_index(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE int32_t google_protobuf_FieldDescriptorProto_oneof_index(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(28, 28)); } +UPB_INLINE bool google_protobuf_FieldDescriptorProto_has_json_name(const google_protobuf_FieldDescriptorProto *msg) { return _upb_has_field(msg, 9); } +UPB_INLINE upb_strview google_protobuf_FieldDescriptorProto_json_name(const google_protobuf_FieldDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(64, 96)); } + +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_name(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_extendee(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_number(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_label(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_type(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_type_name(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 7); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_default_value(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 8); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(56, 80)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_options(google_protobuf_FieldDescriptorProto *msg, google_protobuf_FieldOptions* value) { + _upb_sethas(msg, 10); + UPB_FIELD_AT(msg, google_protobuf_FieldOptions*, UPB_SIZE(72, 112)) = value; +} +UPB_INLINE struct google_protobuf_FieldOptions* google_protobuf_FieldDescriptorProto_mutable_options(google_protobuf_FieldDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_FieldOptions* sub = (struct google_protobuf_FieldOptions*)google_protobuf_FieldDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_FieldOptions*)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_FieldDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_oneof_index(google_protobuf_FieldDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(28, 28)) = value; +} +UPB_INLINE void google_protobuf_FieldDescriptorProto_set_json_name(google_protobuf_FieldDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 9); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(64, 96)) = value; +} + +/* google.protobuf.OneofDescriptorProto */ + +UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_OneofDescriptorProto *)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_OneofDescriptorProto *ret = google_protobuf_OneofDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_OneofDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_OneofDescriptorProto_serialize(const google_protobuf_OneofDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_OneofDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_OneofDescriptorProto_has_name(const google_protobuf_OneofDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_OneofDescriptorProto_name(const google_protobuf_OneofDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_OneofDescriptorProto_has_options(const google_protobuf_OneofDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_OneofOptions* google_protobuf_OneofDescriptorProto_options(const google_protobuf_OneofDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_OneofOptions*, UPB_SIZE(12, 24)); } + +UPB_INLINE void google_protobuf_OneofDescriptorProto_set_name(google_protobuf_OneofDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_OneofDescriptorProto_set_options(google_protobuf_OneofDescriptorProto *msg, google_protobuf_OneofOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_OneofOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_OneofOptions* google_protobuf_OneofDescriptorProto_mutable_options(google_protobuf_OneofDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_OneofOptions* sub = (struct google_protobuf_OneofOptions*)google_protobuf_OneofDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_OneofOptions*)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_OneofDescriptorProto_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.EnumDescriptorProto */ + +UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto *)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumDescriptorProto *ret = google_protobuf_EnumDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumDescriptorProto_serialize(const google_protobuf_EnumDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_name(const google_protobuf_EnumDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_EnumDescriptorProto_name(const google_protobuf_EnumDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE const google_protobuf_EnumValueDescriptorProto* const* google_protobuf_EnumDescriptorProto_value(const google_protobuf_EnumDescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumValueDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_options(const google_protobuf_EnumDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_EnumOptions* google_protobuf_EnumDescriptorProto_options(const google_protobuf_EnumDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_EnumOptions*, UPB_SIZE(12, 24)); } +UPB_INLINE const google_protobuf_EnumDescriptorProto_EnumReservedRange* const* google_protobuf_EnumDescriptorProto_reserved_range(const google_protobuf_EnumDescriptorProto *msg, size_t *len) { return (const google_protobuf_EnumDescriptorProto_EnumReservedRange* const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } +UPB_INLINE upb_strview const* google_protobuf_EnumDescriptorProto_reserved_name(const google_protobuf_EnumDescriptorProto *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(24, 48), len); } + +UPB_INLINE void google_protobuf_EnumDescriptorProto_set_name(google_protobuf_EnumDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE google_protobuf_EnumValueDescriptorProto** google_protobuf_EnumDescriptorProto_mutable_value(google_protobuf_EnumDescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumValueDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE google_protobuf_EnumValueDescriptorProto** google_protobuf_EnumDescriptorProto_resize_value(google_protobuf_EnumDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumValueDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumValueDescriptorProto* google_protobuf_EnumDescriptorProto_add_value(google_protobuf_EnumDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumValueDescriptorProto* sub = (struct google_protobuf_EnumValueDescriptorProto*)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_EnumDescriptorProto_set_options(google_protobuf_EnumDescriptorProto *msg, google_protobuf_EnumOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_EnumOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_EnumOptions* google_protobuf_EnumDescriptorProto_mutable_options(google_protobuf_EnumDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumOptions* sub = (struct google_protobuf_EnumOptions*)google_protobuf_EnumDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_EnumOptions*)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_EnumDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange** google_protobuf_EnumDescriptorProto_mutable_reserved_range(google_protobuf_EnumDescriptorProto *msg, size_t *len) { + return (google_protobuf_EnumDescriptorProto_EnumReservedRange**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange** google_protobuf_EnumDescriptorProto_resize_reserved_range(google_protobuf_EnumDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto_EnumReservedRange**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_EnumDescriptorProto_EnumReservedRange* google_protobuf_EnumDescriptorProto_add_reserved_range(google_protobuf_EnumDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumDescriptorProto_EnumReservedRange* sub = (struct google_protobuf_EnumDescriptorProto_EnumReservedRange*)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE upb_strview* google_protobuf_EnumDescriptorProto_mutable_reserved_name(google_protobuf_EnumDescriptorProto *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(24, 48), len); +} +UPB_INLINE upb_strview* google_protobuf_EnumDescriptorProto_resize_reserved_name(google_protobuf_EnumDescriptorProto *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(24, 48), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_EnumDescriptorProto_add_reserved_name(google_protobuf_EnumDescriptorProto *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(24, 48), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} + +/* google.protobuf.EnumDescriptorProto.EnumReservedRange */ + +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_new(upb_arena *arena) { + return (google_protobuf_EnumDescriptorProto_EnumReservedRange *)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); +} +UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumDescriptorProto_EnumReservedRange *ret = google_protobuf_EnumDescriptorProto_EnumReservedRange_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumDescriptorProto_EnumReservedRange_serialize(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumDescriptorProto_EnumReservedRange_has_start(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_EnumDescriptorProto_EnumReservedRange_start(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_EnumDescriptorProto_EnumReservedRange_has_end(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_EnumDescriptorProto_EnumReservedRange_end(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_start(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_end(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +/* google.protobuf.EnumValueDescriptorProto */ + +UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_EnumValueDescriptorProto *)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumValueDescriptorProto *ret = google_protobuf_EnumValueDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumValueDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumValueDescriptorProto_serialize(const google_protobuf_EnumValueDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumValueDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumValueDescriptorProto_has_name(const google_protobuf_EnumValueDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_EnumValueDescriptorProto_name(const google_protobuf_EnumValueDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_EnumValueDescriptorProto_has_number(const google_protobuf_EnumValueDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_EnumValueDescriptorProto_number(const google_protobuf_EnumValueDescriptorProto *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_EnumValueDescriptorProto_has_options(const google_protobuf_EnumValueDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE const google_protobuf_EnumValueOptions* google_protobuf_EnumValueDescriptorProto_options(const google_protobuf_EnumValueDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_EnumValueOptions*, UPB_SIZE(16, 24)); } + +UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_name(google_protobuf_EnumValueDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_number(google_protobuf_EnumValueDescriptorProto *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_options(google_protobuf_EnumValueDescriptorProto *msg, google_protobuf_EnumValueOptions* value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, google_protobuf_EnumValueOptions*, UPB_SIZE(16, 24)) = value; +} +UPB_INLINE struct google_protobuf_EnumValueOptions* google_protobuf_EnumValueDescriptorProto_mutable_options(google_protobuf_EnumValueDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_EnumValueOptions* sub = (struct google_protobuf_EnumValueOptions*)google_protobuf_EnumValueDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_EnumValueOptions*)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_EnumValueDescriptorProto_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.ServiceDescriptorProto */ + +UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_ServiceDescriptorProto *)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_ServiceDescriptorProto *ret = google_protobuf_ServiceDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_ServiceDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_ServiceDescriptorProto_serialize(const google_protobuf_ServiceDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_ServiceDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_ServiceDescriptorProto_has_name(const google_protobuf_ServiceDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_ServiceDescriptorProto_name(const google_protobuf_ServiceDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE const google_protobuf_MethodDescriptorProto* const* google_protobuf_ServiceDescriptorProto_method(const google_protobuf_ServiceDescriptorProto *msg, size_t *len) { return (const google_protobuf_MethodDescriptorProto* const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE bool google_protobuf_ServiceDescriptorProto_has_options(const google_protobuf_ServiceDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE const google_protobuf_ServiceOptions* google_protobuf_ServiceDescriptorProto_options(const google_protobuf_ServiceDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_ServiceOptions*, UPB_SIZE(12, 24)); } + +UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_name(google_protobuf_ServiceDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE google_protobuf_MethodDescriptorProto** google_protobuf_ServiceDescriptorProto_mutable_method(google_protobuf_ServiceDescriptorProto *msg, size_t *len) { + return (google_protobuf_MethodDescriptorProto**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE google_protobuf_MethodDescriptorProto** google_protobuf_ServiceDescriptorProto_resize_method(google_protobuf_ServiceDescriptorProto *msg, size_t len, upb_arena *arena) { + return (google_protobuf_MethodDescriptorProto**)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_MethodDescriptorProto* google_protobuf_ServiceDescriptorProto_add_method(google_protobuf_ServiceDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_MethodDescriptorProto* sub = (struct google_protobuf_MethodDescriptorProto*)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_options(google_protobuf_ServiceDescriptorProto *msg, google_protobuf_ServiceOptions* value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, google_protobuf_ServiceOptions*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_protobuf_ServiceOptions* google_protobuf_ServiceDescriptorProto_mutable_options(google_protobuf_ServiceDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_ServiceOptions* sub = (struct google_protobuf_ServiceOptions*)google_protobuf_ServiceDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_ServiceOptions*)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_ServiceDescriptorProto_set_options(msg, sub); + } + return sub; +} + +/* google.protobuf.MethodDescriptorProto */ + +UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_new(upb_arena *arena) { + return (google_protobuf_MethodDescriptorProto *)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); +} +UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_MethodDescriptorProto *ret = google_protobuf_MethodDescriptorProto_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_MethodDescriptorProto_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_MethodDescriptorProto_serialize(const google_protobuf_MethodDescriptorProto *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_MethodDescriptorProto_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_name(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE upb_strview google_protobuf_MethodDescriptorProto_name(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_input_type(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE upb_strview google_protobuf_MethodDescriptorProto_input_type(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_output_type(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE upb_strview google_protobuf_MethodDescriptorProto_output_type(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_options(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE const google_protobuf_MethodOptions* google_protobuf_MethodDescriptorProto_options(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, const google_protobuf_MethodOptions*, UPB_SIZE(28, 56)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_client_streaming(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_client_streaming(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_has_server_streaming(const google_protobuf_MethodDescriptorProto *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_MethodDescriptorProto_server_streaming(const google_protobuf_MethodDescriptorProto *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)); } + +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_name(google_protobuf_MethodDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_input_type(google_protobuf_MethodDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_output_type(google_protobuf_MethodDescriptorProto *msg, upb_strview value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(20, 40)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_options(google_protobuf_MethodDescriptorProto *msg, google_protobuf_MethodOptions* value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, google_protobuf_MethodOptions*, UPB_SIZE(28, 56)) = value; +} +UPB_INLINE struct google_protobuf_MethodOptions* google_protobuf_MethodDescriptorProto_mutable_options(google_protobuf_MethodDescriptorProto *msg, upb_arena *arena) { + struct google_protobuf_MethodOptions* sub = (struct google_protobuf_MethodOptions*)google_protobuf_MethodDescriptorProto_options(msg); + if (sub == NULL) { + sub = (struct google_protobuf_MethodOptions*)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); + if (!sub) return NULL; + google_protobuf_MethodDescriptorProto_set_options(msg, sub); + } + return sub; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_client_streaming(google_protobuf_MethodDescriptorProto *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE void google_protobuf_MethodDescriptorProto_set_server_streaming(google_protobuf_MethodDescriptorProto *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)) = value; +} + +/* google.protobuf.FileOptions */ + +UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_new(upb_arena *arena) { + return (google_protobuf_FileOptions *)upb_msg_new(&google_protobuf_FileOptions_msginit, arena); +} +UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FileOptions *ret = google_protobuf_FileOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FileOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FileOptions_serialize(const google_protobuf_FileOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FileOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FileOptions_has_java_package(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 11); } +UPB_INLINE upb_strview google_protobuf_FileOptions_java_package(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(28, 32)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_outer_classname(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 12); } +UPB_INLINE upb_strview google_protobuf_FileOptions_java_outer_classname(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(36, 48)); } +UPB_INLINE bool google_protobuf_FileOptions_has_optimize_for(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_FileOptions_optimize_for(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_multiple_files(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_FileOptions_java_multiple_files(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_FileOptions_has_go_package(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 13); } +UPB_INLINE upb_strview google_protobuf_FileOptions_go_package(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(44, 64)); } +UPB_INLINE bool google_protobuf_FileOptions_has_cc_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE bool google_protobuf_FileOptions_cc_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(17, 17)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE bool google_protobuf_FileOptions_java_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(18, 18)); } +UPB_INLINE bool google_protobuf_FileOptions_has_py_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE bool google_protobuf_FileOptions_py_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(19, 19)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_generate_equals_and_hash(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE bool google_protobuf_FileOptions_java_generate_equals_and_hash(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(20, 20)); } +UPB_INLINE bool google_protobuf_FileOptions_has_deprecated(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 7); } +UPB_INLINE bool google_protobuf_FileOptions_deprecated(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(21, 21)); } +UPB_INLINE bool google_protobuf_FileOptions_has_java_string_check_utf8(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 8); } +UPB_INLINE bool google_protobuf_FileOptions_java_string_check_utf8(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(22, 22)); } +UPB_INLINE bool google_protobuf_FileOptions_has_cc_enable_arenas(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 9); } +UPB_INLINE bool google_protobuf_FileOptions_cc_enable_arenas(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(23, 23)); } +UPB_INLINE bool google_protobuf_FileOptions_has_objc_class_prefix(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 14); } +UPB_INLINE upb_strview google_protobuf_FileOptions_objc_class_prefix(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(52, 80)); } +UPB_INLINE bool google_protobuf_FileOptions_has_csharp_namespace(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 15); } +UPB_INLINE upb_strview google_protobuf_FileOptions_csharp_namespace(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(60, 96)); } +UPB_INLINE bool google_protobuf_FileOptions_has_swift_prefix(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 16); } +UPB_INLINE upb_strview google_protobuf_FileOptions_swift_prefix(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(68, 112)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_class_prefix(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 17); } +UPB_INLINE upb_strview google_protobuf_FileOptions_php_class_prefix(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(76, 128)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_namespace(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 18); } +UPB_INLINE upb_strview google_protobuf_FileOptions_php_namespace(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(84, 144)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_generic_services(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 10); } +UPB_INLINE bool google_protobuf_FileOptions_php_generic_services(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_FileOptions_has_php_metadata_namespace(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 19); } +UPB_INLINE upb_strview google_protobuf_FileOptions_php_metadata_namespace(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(92, 160)); } +UPB_INLINE bool google_protobuf_FileOptions_has_ruby_package(const google_protobuf_FileOptions *msg) { return _upb_has_field(msg, 20); } +UPB_INLINE upb_strview google_protobuf_FileOptions_ruby_package(const google_protobuf_FileOptions *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(100, 176)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_FileOptions_uninterpreted_option(const google_protobuf_FileOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(108, 192), len); } + +UPB_INLINE void google_protobuf_FileOptions_set_java_package(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 11); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(28, 32)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_outer_classname(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 12); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(36, 48)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_optimize_for(google_protobuf_FileOptions *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_multiple_files(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_go_package(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 13); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(44, 64)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_cc_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, bool, UPB_SIZE(17, 17)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, bool, UPB_SIZE(18, 18)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_py_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, bool, UPB_SIZE(19, 19)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_generate_equals_and_hash(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, bool, UPB_SIZE(20, 20)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_deprecated(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 7); + UPB_FIELD_AT(msg, bool, UPB_SIZE(21, 21)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_java_string_check_utf8(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 8); + UPB_FIELD_AT(msg, bool, UPB_SIZE(22, 22)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_cc_enable_arenas(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 9); + UPB_FIELD_AT(msg, bool, UPB_SIZE(23, 23)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_objc_class_prefix(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 14); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(52, 80)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_csharp_namespace(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 15); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(60, 96)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_swift_prefix(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 16); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(68, 112)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_class_prefix(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 17); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(76, 128)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_namespace(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 18); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(84, 144)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_generic_services(google_protobuf_FileOptions *msg, bool value) { + _upb_sethas(msg, 10); + UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_php_metadata_namespace(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 19); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(92, 160)) = value; +} +UPB_INLINE void google_protobuf_FileOptions_set_ruby_package(google_protobuf_FileOptions *msg, upb_strview value) { + _upb_sethas(msg, 20); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(100, 176)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FileOptions_mutable_uninterpreted_option(google_protobuf_FileOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(108, 192), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FileOptions_resize_uninterpreted_option(google_protobuf_FileOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(108, 192), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_FileOptions_add_uninterpreted_option(google_protobuf_FileOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(108, 192), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.MessageOptions */ + +UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_new(upb_arena *arena) { + return (google_protobuf_MessageOptions *)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); +} +UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_MessageOptions *ret = google_protobuf_MessageOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_MessageOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_MessageOptions_serialize(const google_protobuf_MessageOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_MessageOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_MessageOptions_has_message_set_wire_format(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_MessageOptions_message_set_wire_format(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE bool google_protobuf_MessageOptions_has_no_standard_descriptor_accessor(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_MessageOptions_no_standard_descriptor_accessor(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)); } +UPB_INLINE bool google_protobuf_MessageOptions_has_deprecated(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE bool google_protobuf_MessageOptions_deprecated(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(3, 3)); } +UPB_INLINE bool google_protobuf_MessageOptions_has_map_entry(const google_protobuf_MessageOptions *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE bool google_protobuf_MessageOptions_map_entry(const google_protobuf_MessageOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(4, 4)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_MessageOptions_uninterpreted_option(const google_protobuf_MessageOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(8, 8), len); } + +UPB_INLINE void google_protobuf_MessageOptions_set_message_set_wire_format(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE void google_protobuf_MessageOptions_set_no_standard_descriptor_accessor(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)) = value; +} +UPB_INLINE void google_protobuf_MessageOptions_set_deprecated(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, bool, UPB_SIZE(3, 3)) = value; +} +UPB_INLINE void google_protobuf_MessageOptions_set_map_entry(google_protobuf_MessageOptions *msg, bool value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, bool, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MessageOptions_mutable_uninterpreted_option(google_protobuf_MessageOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(8, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MessageOptions_resize_uninterpreted_option(google_protobuf_MessageOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(8, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_MessageOptions_add_uninterpreted_option(google_protobuf_MessageOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(8, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.FieldOptions */ + +UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_new(upb_arena *arena) { + return (google_protobuf_FieldOptions *)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); +} +UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_FieldOptions *ret = google_protobuf_FieldOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_FieldOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_FieldOptions_serialize(const google_protobuf_FieldOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_FieldOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_FieldOptions_has_ctype(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_FieldOptions_ctype(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_packed(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE bool google_protobuf_FieldOptions_packed(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_deprecated(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE bool google_protobuf_FieldOptions_deprecated(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(25, 25)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_lazy(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE bool google_protobuf_FieldOptions_lazy(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(26, 26)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_jstype(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_FieldOptions_jstype(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_FieldOptions_has_weak(const google_protobuf_FieldOptions *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE bool google_protobuf_FieldOptions_weak(const google_protobuf_FieldOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(27, 27)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_FieldOptions_uninterpreted_option(const google_protobuf_FieldOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(28, 32), len); } + +UPB_INLINE void google_protobuf_FieldOptions_set_ctype(google_protobuf_FieldOptions *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_packed(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, bool, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_deprecated(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, bool, UPB_SIZE(25, 25)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_lazy(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, bool, UPB_SIZE(26, 26)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_jstype(google_protobuf_FieldOptions *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_FieldOptions_set_weak(google_protobuf_FieldOptions *msg, bool value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, bool, UPB_SIZE(27, 27)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FieldOptions_mutable_uninterpreted_option(google_protobuf_FieldOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(28, 32), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FieldOptions_resize_uninterpreted_option(google_protobuf_FieldOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(28, 32), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_FieldOptions_add_uninterpreted_option(google_protobuf_FieldOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(28, 32), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.OneofOptions */ + +UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_new(upb_arena *arena) { + return (google_protobuf_OneofOptions *)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); +} +UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_OneofOptions *ret = google_protobuf_OneofOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_OneofOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_OneofOptions_serialize(const google_protobuf_OneofOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_OneofOptions_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_OneofOptions_uninterpreted_option(const google_protobuf_OneofOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_OneofOptions_mutable_uninterpreted_option(google_protobuf_OneofOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_OneofOptions_resize_uninterpreted_option(google_protobuf_OneofOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_OneofOptions_add_uninterpreted_option(google_protobuf_OneofOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.EnumOptions */ + +UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_new(upb_arena *arena) { + return (google_protobuf_EnumOptions *)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); +} +UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumOptions *ret = google_protobuf_EnumOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumOptions_serialize(const google_protobuf_EnumOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumOptions_has_allow_alias(const google_protobuf_EnumOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_EnumOptions_allow_alias(const google_protobuf_EnumOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE bool google_protobuf_EnumOptions_has_deprecated(const google_protobuf_EnumOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_EnumOptions_deprecated(const google_protobuf_EnumOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_EnumOptions_uninterpreted_option(const google_protobuf_EnumOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } + +UPB_INLINE void google_protobuf_EnumOptions_set_allow_alias(google_protobuf_EnumOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE void google_protobuf_EnumOptions_set_deprecated(google_protobuf_EnumOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(2, 2)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumOptions_mutable_uninterpreted_option(google_protobuf_EnumOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumOptions_resize_uninterpreted_option(google_protobuf_EnumOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_EnumOptions_add_uninterpreted_option(google_protobuf_EnumOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.EnumValueOptions */ + +UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_new(upb_arena *arena) { + return (google_protobuf_EnumValueOptions *)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); +} +UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_EnumValueOptions *ret = google_protobuf_EnumValueOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_EnumValueOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_EnumValueOptions_serialize(const google_protobuf_EnumValueOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_EnumValueOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_EnumValueOptions_has_deprecated(const google_protobuf_EnumValueOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_EnumValueOptions_deprecated(const google_protobuf_EnumValueOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_EnumValueOptions_uninterpreted_option(const google_protobuf_EnumValueOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } + +UPB_INLINE void google_protobuf_EnumValueOptions_set_deprecated(google_protobuf_EnumValueOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumValueOptions_mutable_uninterpreted_option(google_protobuf_EnumValueOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_EnumValueOptions_resize_uninterpreted_option(google_protobuf_EnumValueOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_EnumValueOptions_add_uninterpreted_option(google_protobuf_EnumValueOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.ServiceOptions */ + +UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_new(upb_arena *arena) { + return (google_protobuf_ServiceOptions *)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); +} +UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_ServiceOptions *ret = google_protobuf_ServiceOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_ServiceOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_ServiceOptions_serialize(const google_protobuf_ServiceOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_ServiceOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_ServiceOptions_has_deprecated(const google_protobuf_ServiceOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_ServiceOptions_deprecated(const google_protobuf_ServiceOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_ServiceOptions_uninterpreted_option(const google_protobuf_ServiceOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } + +UPB_INLINE void google_protobuf_ServiceOptions_set_deprecated(google_protobuf_ServiceOptions *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ServiceOptions_mutable_uninterpreted_option(google_protobuf_ServiceOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_ServiceOptions_resize_uninterpreted_option(google_protobuf_ServiceOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_ServiceOptions_add_uninterpreted_option(google_protobuf_ServiceOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.MethodOptions */ + +UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_new(upb_arena *arena) { + return (google_protobuf_MethodOptions *)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); +} +UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_MethodOptions *ret = google_protobuf_MethodOptions_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_MethodOptions_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_MethodOptions_serialize(const google_protobuf_MethodOptions *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_MethodOptions_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_MethodOptions_has_deprecated(const google_protobuf_MethodOptions *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE bool google_protobuf_MethodOptions_deprecated(const google_protobuf_MethodOptions *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_MethodOptions_has_idempotency_level(const google_protobuf_MethodOptions *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_MethodOptions_idempotency_level(const google_protobuf_MethodOptions *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } +UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_MethodOptions_uninterpreted_option(const google_protobuf_MethodOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(20, 24), len); } + +UPB_INLINE void google_protobuf_MethodOptions_set_deprecated(google_protobuf_MethodOptions *msg, bool value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, bool, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_MethodOptions_set_idempotency_level(google_protobuf_MethodOptions *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MethodOptions_mutable_uninterpreted_option(google_protobuf_MethodOptions *msg, size_t *len) { + return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 24), len); +} +UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_MethodOptions_resize_uninterpreted_option(google_protobuf_MethodOptions *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 24), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_MethodOptions_add_uninterpreted_option(google_protobuf_MethodOptions *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 24), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.UninterpretedOption */ + +UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_new(upb_arena *arena) { + return (google_protobuf_UninterpretedOption *)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); +} +UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_UninterpretedOption *ret = google_protobuf_UninterpretedOption_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_UninterpretedOption_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_UninterpretedOption_serialize(const google_protobuf_UninterpretedOption *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_UninterpretedOption_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_UninterpretedOption_NamePart* const* google_protobuf_UninterpretedOption_name(const google_protobuf_UninterpretedOption *msg, size_t *len) { return (const google_protobuf_UninterpretedOption_NamePart* const*)_upb_array_accessor(msg, UPB_SIZE(56, 80), len); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_identifier_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 4); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_identifier_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_positive_int_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE uint64_t google_protobuf_UninterpretedOption_positive_int_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, uint64_t, UPB_SIZE(8, 8)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_negative_int_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int64_t google_protobuf_UninterpretedOption_negative_int_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, int64_t, UPB_SIZE(16, 16)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_double_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE double google_protobuf_UninterpretedOption_double_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, double, UPB_SIZE(24, 24)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_string_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 5); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_string_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_has_aggregate_value(const google_protobuf_UninterpretedOption *msg) { return _upb_has_field(msg, 6); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_aggregate_value(const google_protobuf_UninterpretedOption *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)); } + +UPB_INLINE google_protobuf_UninterpretedOption_NamePart** google_protobuf_UninterpretedOption_mutable_name(google_protobuf_UninterpretedOption *msg, size_t *len) { + return (google_protobuf_UninterpretedOption_NamePart**)_upb_array_mutable_accessor(msg, UPB_SIZE(56, 80), len); +} +UPB_INLINE google_protobuf_UninterpretedOption_NamePart** google_protobuf_UninterpretedOption_resize_name(google_protobuf_UninterpretedOption *msg, size_t len, upb_arena *arena) { + return (google_protobuf_UninterpretedOption_NamePart**)_upb_array_resize_accessor(msg, UPB_SIZE(56, 80), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_UninterpretedOption_NamePart* google_protobuf_UninterpretedOption_add_name(google_protobuf_UninterpretedOption *msg, upb_arena *arena) { + struct google_protobuf_UninterpretedOption_NamePart* sub = (struct google_protobuf_UninterpretedOption_NamePart*)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(56, 80), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_identifier_value(google_protobuf_UninterpretedOption *msg, upb_strview value) { + _upb_sethas(msg, 4); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(32, 32)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_positive_int_value(google_protobuf_UninterpretedOption *msg, uint64_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, uint64_t, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_negative_int_value(google_protobuf_UninterpretedOption *msg, int64_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int64_t, UPB_SIZE(16, 16)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_double_value(google_protobuf_UninterpretedOption *msg, double value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, double, UPB_SIZE(24, 24)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_string_value(google_protobuf_UninterpretedOption *msg, upb_strview value) { + _upb_sethas(msg, 5); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(40, 48)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_set_aggregate_value(google_protobuf_UninterpretedOption *msg, upb_strview value) { + _upb_sethas(msg, 6); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(48, 64)) = value; +} + +/* google.protobuf.UninterpretedOption.NamePart */ + +UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_new(upb_arena *arena) { + return (google_protobuf_UninterpretedOption_NamePart *)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); +} +UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_UninterpretedOption_NamePart *ret = google_protobuf_UninterpretedOption_NamePart_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_UninterpretedOption_NamePart_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_UninterpretedOption_NamePart_serialize(const google_protobuf_UninterpretedOption_NamePart *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_UninterpretedOption_NamePart_msginit, arena, len); +} + +UPB_INLINE bool google_protobuf_UninterpretedOption_NamePart_has_name_part(const google_protobuf_UninterpretedOption_NamePart *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_UninterpretedOption_NamePart_name_part(const google_protobuf_UninterpretedOption_NamePart *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_UninterpretedOption_NamePart_has_is_extension(const google_protobuf_UninterpretedOption_NamePart *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE bool google_protobuf_UninterpretedOption_NamePart_is_extension(const google_protobuf_UninterpretedOption_NamePart *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)); } + +UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_name_part(google_protobuf_UninterpretedOption_NamePart *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_is_extension(google_protobuf_UninterpretedOption_NamePart *msg, bool value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, bool, UPB_SIZE(1, 1)) = value; +} + +/* google.protobuf.SourceCodeInfo */ + +UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_new(upb_arena *arena) { + return (google_protobuf_SourceCodeInfo *)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); +} +UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_SourceCodeInfo *ret = google_protobuf_SourceCodeInfo_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_SourceCodeInfo_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_SourceCodeInfo_serialize(const google_protobuf_SourceCodeInfo *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_SourceCodeInfo_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_SourceCodeInfo_Location* const* google_protobuf_SourceCodeInfo_location(const google_protobuf_SourceCodeInfo *msg, size_t *len) { return (const google_protobuf_SourceCodeInfo_Location* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_SourceCodeInfo_Location** google_protobuf_SourceCodeInfo_mutable_location(google_protobuf_SourceCodeInfo *msg, size_t *len) { + return (google_protobuf_SourceCodeInfo_Location**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_SourceCodeInfo_Location** google_protobuf_SourceCodeInfo_resize_location(google_protobuf_SourceCodeInfo *msg, size_t len, upb_arena *arena) { + return (google_protobuf_SourceCodeInfo_Location**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_SourceCodeInfo_Location* google_protobuf_SourceCodeInfo_add_location(google_protobuf_SourceCodeInfo *msg, upb_arena *arena) { + struct google_protobuf_SourceCodeInfo_Location* sub = (struct google_protobuf_SourceCodeInfo_Location*)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.SourceCodeInfo.Location */ + +UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_new(upb_arena *arena) { + return (google_protobuf_SourceCodeInfo_Location *)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); +} +UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_SourceCodeInfo_Location *ret = google_protobuf_SourceCodeInfo_Location_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_SourceCodeInfo_Location_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_SourceCodeInfo_Location_serialize(const google_protobuf_SourceCodeInfo_Location *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_SourceCodeInfo_Location_msginit, arena, len); +} + +UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_path(const google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } +UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_span(const google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(24, 48), len); } +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_leading_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE upb_strview google_protobuf_SourceCodeInfo_Location_leading_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_trailing_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE upb_strview google_protobuf_SourceCodeInfo_Location_trailing_comments(const google_protobuf_SourceCodeInfo_Location *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)); } +UPB_INLINE upb_strview const* google_protobuf_SourceCodeInfo_Location_leading_detached_comments(const google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { return (upb_strview const*)_upb_array_accessor(msg, UPB_SIZE(28, 56), len); } + +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_mutable_path(google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_resize_path(google_protobuf_SourceCodeInfo_Location *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_add_path(google_protobuf_SourceCodeInfo_Location *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_mutable_span(google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(24, 48), len); +} +UPB_INLINE int32_t* google_protobuf_SourceCodeInfo_Location_resize_span(google_protobuf_SourceCodeInfo_Location *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(24, 48), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_add_span(google_protobuf_SourceCodeInfo_Location *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(24, 48), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_leading_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_strview value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_trailing_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_strview value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE upb_strview* google_protobuf_SourceCodeInfo_Location_mutable_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, size_t *len) { + return (upb_strview*)_upb_array_mutable_accessor(msg, UPB_SIZE(28, 56), len); +} +UPB_INLINE upb_strview* google_protobuf_SourceCodeInfo_Location_resize_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, size_t len, upb_arena *arena) { + return (upb_strview*)_upb_array_resize_accessor(msg, UPB_SIZE(28, 56), len, UPB_SIZE(8, 16), UPB_TYPE_STRING, arena); +} +UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_add_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_strview val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(28, 56), UPB_SIZE(8, 16), UPB_TYPE_STRING, &val, arena); +} + +/* google.protobuf.GeneratedCodeInfo */ + +UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_new(upb_arena *arena) { + return (google_protobuf_GeneratedCodeInfo *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena); +} +UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_GeneratedCodeInfo *ret = google_protobuf_GeneratedCodeInfo_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_GeneratedCodeInfo_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_GeneratedCodeInfo_serialize(const google_protobuf_GeneratedCodeInfo *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_GeneratedCodeInfo_msginit, arena, len); +} + +UPB_INLINE const google_protobuf_GeneratedCodeInfo_Annotation* const* google_protobuf_GeneratedCodeInfo_annotation(const google_protobuf_GeneratedCodeInfo *msg, size_t *len) { return (const google_protobuf_GeneratedCodeInfo_Annotation* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation** google_protobuf_GeneratedCodeInfo_mutable_annotation(google_protobuf_GeneratedCodeInfo *msg, size_t *len) { + return (google_protobuf_GeneratedCodeInfo_Annotation**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation** google_protobuf_GeneratedCodeInfo_resize_annotation(google_protobuf_GeneratedCodeInfo *msg, size_t len, upb_arena *arena) { + return (google_protobuf_GeneratedCodeInfo_Annotation**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_protobuf_GeneratedCodeInfo_Annotation* google_protobuf_GeneratedCodeInfo_add_annotation(google_protobuf_GeneratedCodeInfo *msg, upb_arena *arena) { + struct google_protobuf_GeneratedCodeInfo_Annotation* sub = (struct google_protobuf_GeneratedCodeInfo_Annotation*)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.protobuf.GeneratedCodeInfo.Annotation */ + +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_new(upb_arena *arena) { + return (google_protobuf_GeneratedCodeInfo_Annotation *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); +} +UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_parse(const char *buf, size_t size, + upb_arena *arena) { + google_protobuf_GeneratedCodeInfo_Annotation *ret = google_protobuf_GeneratedCodeInfo_Annotation_new(arena); + return (ret && upb_decode(buf, size, ret, &google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_protobuf_GeneratedCodeInfo_Annotation_serialize(const google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena, len); +} + +UPB_INLINE int32_t const* google_protobuf_GeneratedCodeInfo_Annotation_path(const google_protobuf_GeneratedCodeInfo_Annotation *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(20, 32), len); } +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_source_file(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return _upb_has_field(msg, 3); } +UPB_INLINE upb_strview google_protobuf_GeneratedCodeInfo_Annotation_source_file(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 16)); } +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_begin(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return _upb_has_field(msg, 1); } +UPB_INLINE int32_t google_protobuf_GeneratedCodeInfo_Annotation_begin(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_end(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return _upb_has_field(msg, 2); } +UPB_INLINE int32_t google_protobuf_GeneratedCodeInfo_Annotation_end(const google_protobuf_GeneratedCodeInfo_Annotation *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE int32_t* google_protobuf_GeneratedCodeInfo_Annotation_mutable_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 32), len); +} +UPB_INLINE int32_t* google_protobuf_GeneratedCodeInfo_Annotation_resize_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(20, 32), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_add_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(20, 32), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE void google_protobuf_GeneratedCodeInfo_Annotation_set_source_file(google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_strview value) { + _upb_sethas(msg, 3); + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 16)) = value; +} +UPB_INLINE void google_protobuf_GeneratedCodeInfo_Annotation_set_begin(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t value) { + _upb_sethas(msg, 1); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_protobuf_GeneratedCodeInfo_Annotation_set_end(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t value) { + _upb_sethas(msg, 2); + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ */ diff --git a/generated_for_cmake/upb/json/parser.c b/generated_for_cmake/upb/json/parser.c new file mode 100644 index 00000000000..f72e945881f --- /dev/null +++ b/generated_for_cmake/upb/json/parser.c @@ -0,0 +1,3454 @@ + +#line 1 "upb/json/parser.rl" +/* +** upb::json::Parser (upb_json_parser) +** +** A parser that uses the Ragel State Machine Compiler to generate +** the finite automata. +** +** Ragel only natively handles regular languages, but we can manually +** program it a bit to handle context-free languages like JSON, by using +** the "fcall" and "fret" constructs. +** +** This parser can handle the basics, but needs several things to be fleshed +** out: +** +** - handling of unicode escape sequences (including high surrogate pairs). +** - properly check and report errors for unknown fields, stack overflow, +** improper array nesting (or lack of nesting). +** - handling of base64 sequences with padding characters. +** - handling of push-back (non-success returns from sink functions). +** - handling of keys/escape-sequences/etc that span input buffers. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "upb/json/parser.h" +#include "upb/pb/encoder.h" + +#include "upb/port_def.inc" + +#define UPB_JSON_MAX_DEPTH 64 + +/* Type of value message */ +enum { + VALUE_NULLVALUE = 0, + VALUE_NUMBERVALUE = 1, + VALUE_STRINGVALUE = 2, + VALUE_BOOLVALUE = 3, + VALUE_STRUCTVALUE = 4, + VALUE_LISTVALUE = 5 +}; + +/* Forward declare */ +static bool is_top_level(upb_json_parser *p); +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type); +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type); + +static bool is_number_wrapper_object(upb_json_parser *p); +static bool does_number_wrapper_start(upb_json_parser *p); +static bool does_number_wrapper_end(upb_json_parser *p); + +static bool is_string_wrapper_object(upb_json_parser *p); +static bool does_string_wrapper_start(upb_json_parser *p); +static bool does_string_wrapper_end(upb_json_parser *p); + +static bool does_fieldmask_start(upb_json_parser *p); +static bool does_fieldmask_end(upb_json_parser *p); +static void start_fieldmask_object(upb_json_parser *p); +static void end_fieldmask_object(upb_json_parser *p); + +static void start_wrapper_object(upb_json_parser *p); +static void end_wrapper_object(upb_json_parser *p); + +static void start_value_object(upb_json_parser *p, int value_type); +static void end_value_object(upb_json_parser *p); + +static void start_listvalue_object(upb_json_parser *p); +static void end_listvalue_object(upb_json_parser *p); + +static void start_structvalue_object(upb_json_parser *p); +static void end_structvalue_object(upb_json_parser *p); + +static void start_object(upb_json_parser *p); +static void end_object(upb_json_parser *p); + +static void start_any_object(upb_json_parser *p, const char *ptr); +static bool end_any_object(upb_json_parser *p, const char *ptr); + +static bool start_subobject(upb_json_parser *p); +static void end_subobject(upb_json_parser *p); + +static void start_member(upb_json_parser *p); +static void end_member(upb_json_parser *p); +static bool end_membername(upb_json_parser *p); + +static void start_any_member(upb_json_parser *p, const char *ptr); +static void end_any_member(upb_json_parser *p, const char *ptr); +static bool end_any_membername(upb_json_parser *p); + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle); +static bool end(void *closure, const void *hd); + +static const char eof_ch = 'e'; + +/* stringsink */ +typedef struct { + upb_byteshandler handler; + upb_bytessink sink; + char *ptr; + size_t len, size; +} upb_stringsink; + + +static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) { + upb_stringsink *sink = _sink; + sink->len = 0; + UPB_UNUSED(hd); + UPB_UNUSED(size_hint); + return sink; +} + +static size_t stringsink_string(void *_sink, const void *hd, const char *ptr, + size_t len, const upb_bufhandle *handle) { + upb_stringsink *sink = _sink; + size_t new_size = sink->size; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + while (sink->len + len > new_size) { + new_size *= 2; + } + + if (new_size != sink->size) { + sink->ptr = realloc(sink->ptr, new_size); + sink->size = new_size; + } + + memcpy(sink->ptr + sink->len, ptr, len); + sink->len += len; + + return len; +} + +void upb_stringsink_init(upb_stringsink *sink) { + upb_byteshandler_init(&sink->handler); + upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL); + upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL); + + upb_bytessink_reset(&sink->sink, &sink->handler, sink); + + sink->size = 32; + sink->ptr = malloc(sink->size); + sink->len = 0; +} + +void upb_stringsink_uninit(upb_stringsink *sink) { free(sink->ptr); } + +typedef struct { + /* For encoding Any value field in binary format. */ + upb_handlercache *encoder_handlercache; + upb_stringsink stringsink; + + /* For decoding Any value field in json format. */ + upb_json_codecache *parser_codecache; + upb_sink sink; + upb_json_parser *parser; + + /* Mark the range of uninterpreted values in json input before type url. */ + const char *before_type_url_start; + const char *before_type_url_end; + + /* Mark the range of uninterpreted values in json input after type url. */ + const char *after_type_url_start; +} upb_jsonparser_any_frame; + +typedef struct { + upb_sink sink; + + /* The current message in which we're parsing, and the field whose value we're + * expecting next. */ + const upb_msgdef *m; + const upb_fielddef *f; + + /* The table mapping json name to fielddef for this message. */ + const upb_strtable *name_table; + + /* We are in a repeated-field context. We need this flag to decide whether to + * handle the array as a normal repeated field or a + * google.protobuf.ListValue/google.protobuf.Value. */ + bool is_repeated; + + /* We are in a repeated-field context, ready to emit mapentries as + * submessages. This flag alters the start-of-object (open-brace) behavior to + * begin a sequence of mapentry messages rather than a single submessage. */ + bool is_map; + + /* We are in a map-entry message context. This flag is set when parsing the + * value field of a single map entry and indicates to all value-field parsers + * (subobjects, strings, numbers, and bools) that the map-entry submessage + * should end as soon as the value is parsed. */ + bool is_mapentry; + + /* If |is_map| or |is_mapentry| is true, |mapfield| refers to the parent + * message's map field that we're currently parsing. This differs from |f| + * because |f| is the field in the *current* message (i.e., the map-entry + * message itself), not the parent's field that leads to this map. */ + const upb_fielddef *mapfield; + + /* We are in an Any message context. This flag is set when parsing the Any + * message and indicates to all field parsers (subobjects, strings, numbers, + * and bools) that the parsed field should be serialized as binary data or + * cached (type url not found yet). */ + bool is_any; + + /* The type of packed message in Any. */ + upb_jsonparser_any_frame *any_frame; + + /* True if the field to be parsed is unknown. */ + bool is_unknown_field; +} upb_jsonparser_frame; + +static void init_frame(upb_jsonparser_frame* frame) { + frame->m = NULL; + frame->f = NULL; + frame->name_table = NULL; + frame->is_repeated = false; + frame->is_map = false; + frame->is_mapentry = false; + frame->mapfield = NULL; + frame->is_any = false; + frame->any_frame = NULL; + frame->is_unknown_field = false; +} + +struct upb_json_parser { + upb_arena *arena; + const upb_json_parsermethod *method; + upb_bytessink input_; + + /* Stack to track the JSON scopes we are in. */ + upb_jsonparser_frame stack[UPB_JSON_MAX_DEPTH]; + upb_jsonparser_frame *top; + upb_jsonparser_frame *limit; + + upb_status *status; + + /* Ragel's internal parsing stack for the parsing state machine. */ + int current_state; + int parser_stack[UPB_JSON_MAX_DEPTH]; + int parser_top; + + /* The handle for the current buffer. */ + const upb_bufhandle *handle; + + /* Accumulate buffer. See details in parser.rl. */ + const char *accumulated; + size_t accumulated_len; + char *accumulate_buf; + size_t accumulate_buf_size; + + /* Multi-part text data. See details in parser.rl. */ + int multipart_state; + upb_selector_t string_selector; + + /* Input capture. See details in parser.rl. */ + const char *capture; + + /* Intermediate result of parsing a unicode escape sequence. */ + uint32_t digit; + + /* For resolve type url in Any. */ + const upb_symtab *symtab; + + /* Whether to proceed if unknown field is met. */ + bool ignore_json_unknown; + + /* Cache for parsing timestamp due to base and zone are handled in different + * handlers. */ + struct tm tm; +}; + +static upb_jsonparser_frame* start_jsonparser_frame(upb_json_parser *p) { + upb_jsonparser_frame *inner; + inner = p->top + 1; + init_frame(inner); + return inner; +} + +struct upb_json_codecache { + upb_arena *arena; + upb_inttable methods; /* upb_msgdef* -> upb_json_parsermethod* */ +}; + +struct upb_json_parsermethod { + const upb_json_codecache *cache; + upb_byteshandler input_handler_; + + /* Maps json_name -> fielddef */ + upb_strtable name_table; +}; + +#define PARSER_CHECK_RETURN(x) if (!(x)) return false + +static upb_jsonparser_any_frame *json_parser_any_frame_new( + upb_json_parser *p) { + upb_jsonparser_any_frame *frame; + + frame = upb_arena_malloc(p->arena, sizeof(upb_jsonparser_any_frame)); + + frame->encoder_handlercache = upb_pb_encoder_newcache(); + frame->parser_codecache = upb_json_codecache_new(); + frame->parser = NULL; + frame->before_type_url_start = NULL; + frame->before_type_url_end = NULL; + frame->after_type_url_start = NULL; + + upb_stringsink_init(&frame->stringsink); + + return frame; +} + +static void json_parser_any_frame_set_payload_type( + upb_json_parser *p, + upb_jsonparser_any_frame *frame, + const upb_msgdef *payload_type) { + const upb_handlers *h; + const upb_json_parsermethod *parser_method; + upb_pb_encoder *encoder; + + /* Initialize encoder. */ + h = upb_handlercache_get(frame->encoder_handlercache, payload_type); + encoder = upb_pb_encoder_create(p->arena, h, frame->stringsink.sink); + + /* Initialize parser. */ + parser_method = upb_json_codecache_get(frame->parser_codecache, payload_type); + upb_sink_reset(&frame->sink, h, encoder); + frame->parser = + upb_json_parser_create(p->arena, parser_method, p->symtab, frame->sink, + p->status, p->ignore_json_unknown); +} + +static void json_parser_any_frame_free(upb_jsonparser_any_frame *frame) { + upb_handlercache_free(frame->encoder_handlercache); + upb_json_codecache_free(frame->parser_codecache); + upb_stringsink_uninit(&frame->stringsink); +} + +static bool json_parser_any_frame_has_type_url( + upb_jsonparser_any_frame *frame) { + return frame->parser != NULL; +} + +static bool json_parser_any_frame_has_value_before_type_url( + upb_jsonparser_any_frame *frame) { + return frame->before_type_url_start != frame->before_type_url_end; +} + +static bool json_parser_any_frame_has_value_after_type_url( + upb_jsonparser_any_frame *frame) { + return frame->after_type_url_start != NULL; +} + +static bool json_parser_any_frame_has_value( + upb_jsonparser_any_frame *frame) { + return json_parser_any_frame_has_value_before_type_url(frame) || + json_parser_any_frame_has_value_after_type_url(frame); +} + +static void json_parser_any_frame_set_before_type_url_end( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (frame->parser == NULL) { + frame->before_type_url_end = ptr; + } +} + +static void json_parser_any_frame_set_after_type_url_start_once( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (json_parser_any_frame_has_type_url(frame) && + frame->after_type_url_start == NULL) { + frame->after_type_url_start = ptr; + } +} + +/* Used to signal that a capture has been suspended. */ +static char suspend_capture; + +static upb_selector_t getsel_for_handlertype(upb_json_parser *p, + upb_handlertype_t type) { + upb_selector_t sel; + bool ok = upb_handlers_getselector(p->top->f, type, &sel); + UPB_ASSERT(ok); + return sel; +} + +static upb_selector_t parser_getsel(upb_json_parser *p) { + return getsel_for_handlertype( + p, upb_handlers_getprimitivehandlertype(p->top->f)); +} + +static bool check_stack(upb_json_parser *p) { + if ((p->top + 1) == p->limit) { + upb_status_seterrmsg(p->status, "Nesting too deep"); + return false; + } + + return true; +} + +static void set_name_table(upb_json_parser *p, upb_jsonparser_frame *frame) { + upb_value v; + const upb_json_codecache *cache = p->method->cache; + bool ok; + const upb_json_parsermethod *method; + + ok = upb_inttable_lookupptr(&cache->methods, frame->m, &v); + UPB_ASSERT(ok); + method = upb_value_getconstptr(v); + + frame->name_table = &method->name_table; +} + +/* There are GCC/Clang built-ins for overflow checking which we could start + * using if there was any performance benefit to it. */ + +static bool checked_add(size_t a, size_t b, size_t *c) { + if (SIZE_MAX - a < b) return false; + *c = a + b; + return true; +} + +static size_t saturating_multiply(size_t a, size_t b) { + /* size_t is unsigned, so this is defined behavior even on overflow. */ + size_t ret = a * b; + if (b != 0 && ret / b != a) { + ret = SIZE_MAX; + } + return ret; +} + + +/* Base64 decoding ************************************************************/ + +/* TODO(haberman): make this streaming. */ + +static const signed char b64table[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62/*+*/, -1, -1, -1, 63/*/ */, + 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, + 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, + -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, + 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, + 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1, + -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, + 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/, + 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/, + 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* Returns the table value sign-extended to 32 bits. Knowing that the upper + * bits will be 1 for unrecognized characters makes it easier to check for + * this error condition later (see below). */ +int32_t b64lookup(unsigned char ch) { return b64table[ch]; } + +/* Returns true if the given character is not a valid base64 character or + * padding. */ +bool nonbase64(unsigned char ch) { return b64lookup(ch) == -1 && ch != '='; } + +static bool base64_push(upb_json_parser *p, upb_selector_t sel, const char *ptr, + size_t len) { + const char *limit = ptr + len; + for (; ptr < limit; ptr += 4) { + uint32_t val; + char output[3]; + + if (limit - ptr < 4) { + upb_status_seterrf(p->status, + "Base64 input for bytes field not a multiple of 4: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6 | + b64lookup(ptr[3]); + + /* Test the upper bit; returns true if any of the characters returned -1. */ + if (val & 0x80000000) { + goto otherchar; + } + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + output[2] = val & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 3, NULL); + } + return true; + +otherchar: + if (nonbase64(ptr[0]) || nonbase64(ptr[1]) || nonbase64(ptr[2]) || + nonbase64(ptr[3]) ) { + upb_status_seterrf(p->status, + "Non-base64 characters in bytes field: %s", + upb_fielddef_name(p->top->f)); + return false; + } if (ptr[2] == '=') { + uint32_t val; + char output; + + /* Last group contains only two input bytes, one output byte. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[3] != '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12; + + UPB_ASSERT(!(val & 0x80000000)); + output = val >> 16; + upb_sink_putstring(p->top->sink, sel, &output, 1, NULL); + return true; + } else { + uint32_t val; + char output[2]; + + /* Last group contains only three input bytes, two output bytes. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[2] == '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6; + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 2, NULL); + return true; + } + +badpadding: + upb_status_seterrf(p->status, + "Incorrect base64 padding for field: %s (%.*s)", + upb_fielddef_name(p->top->f), + 4, ptr); + return false; +} + + +/* Accumulate buffer **********************************************************/ + +/* Functionality for accumulating a buffer. + * + * Some parts of the parser need an entire value as a contiguous string. For + * example, to look up a member name in a hash table, or to turn a string into + * a number, the relevant library routines need the input string to be in + * contiguous memory, even if the value spanned two or more buffers in the + * input. These routines handle that. + * + * In the common case we can just point to the input buffer to get this + * contiguous string and avoid any actual copy. So we optimistically begin + * this way. But there are a few cases where we must instead copy into a + * separate buffer: + * + * 1. The string was not contiguous in the input (it spanned buffers). + * + * 2. The string included escape sequences that need to be interpreted to get + * the true value in a contiguous buffer. */ + +static void assert_accumulate_empty(upb_json_parser *p) { + UPB_ASSERT(p->accumulated == NULL); + UPB_ASSERT(p->accumulated_len == 0); +} + +static void accumulate_clear(upb_json_parser *p) { + p->accumulated = NULL; + p->accumulated_len = 0; +} + +/* Used internally by accumulate_append(). */ +static bool accumulate_realloc(upb_json_parser *p, size_t need) { + void *mem; + size_t old_size = p->accumulate_buf_size; + size_t new_size = UPB_MAX(old_size, 128); + while (new_size < need) { + new_size = saturating_multiply(new_size, 2); + } + + mem = upb_arena_realloc(p->arena, p->accumulate_buf, old_size, new_size); + if (!mem) { + upb_status_seterrmsg(p->status, "Out of memory allocating buffer."); + return false; + } + + p->accumulate_buf = mem; + p->accumulate_buf_size = new_size; + return true; +} + +/* Logically appends the given data to the append buffer. + * If "can_alias" is true, we will try to avoid actually copying, but the buffer + * must be valid until the next accumulate_append() call (if any). */ +static bool accumulate_append(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + size_t need; + + if (!p->accumulated && can_alias) { + p->accumulated = buf; + p->accumulated_len = len; + return true; + } + + if (!checked_add(p->accumulated_len, len, &need)) { + upb_status_seterrmsg(p->status, "Integer overflow."); + return false; + } + + if (need > p->accumulate_buf_size && !accumulate_realloc(p, need)) { + return false; + } + + if (p->accumulated != p->accumulate_buf) { + memcpy(p->accumulate_buf, p->accumulated, p->accumulated_len); + p->accumulated = p->accumulate_buf; + } + + memcpy(p->accumulate_buf + p->accumulated_len, buf, len); + p->accumulated_len += len; + return true; +} + +/* Returns a pointer to the data accumulated since the last accumulate_clear() + * call, and writes the length to *len. This with point either to the input + * buffer or a temporary accumulate buffer. */ +static const char *accumulate_getptr(upb_json_parser *p, size_t *len) { + UPB_ASSERT(p->accumulated); + *len = p->accumulated_len; + return p->accumulated; +} + + +/* Mult-part text data ********************************************************/ + +/* When we have text data in the input, it can often come in multiple segments. + * For example, there may be some raw string data followed by an escape + * sequence. The two segments are processed with different logic. Also buffer + * seams in the input can cause multiple segments. + * + * As we see segments, there are two main cases for how we want to process them: + * + * 1. we want to push the captured input directly to string handlers. + * + * 2. we need to accumulate all the parts into a contiguous buffer for further + * processing (field name lookup, string->number conversion, etc). */ + +/* This is the set of states for p->multipart_state. */ +enum { + /* We are not currently processing multipart data. */ + MULTIPART_INACTIVE = 0, + + /* We are processing multipart data by accumulating it into a contiguous + * buffer. */ + MULTIPART_ACCUMULATE = 1, + + /* We are processing multipart data by pushing each part directly to the + * current string handlers. */ + MULTIPART_PUSHEAGERLY = 2 +}; + +/* Start a multi-part text value where we accumulate the data for processing at + * the end. */ +static void multipart_startaccum(upb_json_parser *p) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_ACCUMULATE; +} + +/* Start a multi-part text value where we immediately push text data to a string + * value with the given selector. */ +static void multipart_start(upb_json_parser *p, upb_selector_t sel) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_PUSHEAGERLY; + p->string_selector = sel; +} + +static bool multipart_text(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + switch (p->multipart_state) { + case MULTIPART_INACTIVE: + upb_status_seterrmsg( + p->status, "Internal error: unexpected state MULTIPART_INACTIVE"); + return false; + + case MULTIPART_ACCUMULATE: + if (!accumulate_append(p, buf, len, can_alias)) { + return false; + } + break; + + case MULTIPART_PUSHEAGERLY: { + const upb_bufhandle *handle = can_alias ? p->handle : NULL; + upb_sink_putstring(p->top->sink, p->string_selector, buf, len, handle); + break; + } + } + + return true; +} + +/* Note: this invalidates the accumulate buffer! Call only after reading its + * contents. */ +static void multipart_end(upb_json_parser *p) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_INACTIVE; + accumulate_clear(p); +} + + +/* Input capture **************************************************************/ + +/* Functionality for capturing a region of the input as text. Gracefully + * handles the case where a buffer seam occurs in the middle of the captured + * region. */ + +static void capture_begin(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + UPB_ASSERT(p->capture == NULL); + p->capture = ptr; +} + +static bool capture_end(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->capture); + if (multipart_text(p, p->capture, ptr - p->capture, true)) { + p->capture = NULL; + return true; + } else { + return false; + } +} + +/* This is called at the end of each input buffer (ie. when we have hit a + * buffer seam). If we are in the middle of capturing the input, this + * processes the unprocessed capture region. */ +static void capture_suspend(upb_json_parser *p, const char **ptr) { + if (!p->capture) return; + + if (multipart_text(p, p->capture, *ptr - p->capture, false)) { + /* We use this as a signal that we were in the middle of capturing, and + * that capturing should resume at the beginning of the next buffer. + * + * We can't use *ptr here, because we have no guarantee that this pointer + * will be valid when we resume (if the underlying memory is freed, then + * using the pointer at all, even to compare to NULL, is likely undefined + * behavior). */ + p->capture = &suspend_capture; + } else { + /* Need to back up the pointer to the beginning of the capture, since + * we were not able to actually preserve it. */ + *ptr = p->capture; + } +} + +static void capture_resume(upb_json_parser *p, const char *ptr) { + if (p->capture) { + UPB_ASSERT(p->capture == &suspend_capture); + p->capture = ptr; + } +} + + +/* Callbacks from the parser **************************************************/ + +/* These are the functions called directly from the parser itself. + * We define these in the same order as their declarations in the parser. */ + +static char escape_char(char in) { + switch (in) { + case 'r': return '\r'; + case 't': return '\t'; + case 'n': return '\n'; + case 'f': return '\f'; + case 'b': return '\b'; + case '/': return '/'; + case '"': return '"'; + case '\\': return '\\'; + default: + UPB_ASSERT(0); + return 'x'; + } +} + +static bool escape(upb_json_parser *p, const char *ptr) { + char ch = escape_char(*ptr); + return multipart_text(p, &ch, 1, false); +} + +static void start_hex(upb_json_parser *p) { + p->digit = 0; +} + +static void hexdigit(upb_json_parser *p, const char *ptr) { + char ch = *ptr; + + p->digit <<= 4; + + if (ch >= '0' && ch <= '9') { + p->digit += (ch - '0'); + } else if (ch >= 'a' && ch <= 'f') { + p->digit += ((ch - 'a') + 10); + } else { + UPB_ASSERT(ch >= 'A' && ch <= 'F'); + p->digit += ((ch - 'A') + 10); + } +} + +static bool end_hex(upb_json_parser *p) { + uint32_t codepoint = p->digit; + + /* emit the codepoint as UTF-8. */ + char utf8[3]; /* support \u0000 -- \uFFFF -- need only three bytes. */ + int length = 0; + if (codepoint <= 0x7F) { + utf8[0] = codepoint; + length = 1; + } else if (codepoint <= 0x07FF) { + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x1F) | 0xC0; + length = 2; + } else /* codepoint <= 0xFFFF */ { + utf8[2] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x0F) | 0xE0; + length = 3; + } + /* TODO(haberman): Handle high surrogates: if codepoint is a high surrogate + * we have to wait for the next escape to get the full code point). */ + + return multipart_text(p, utf8, length, false); +} + +static void start_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_number(upb_json_parser *p, const char *ptr) { + if (is_top_level(p)) { + if (is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NUMBERVALUE); + } else { + return false; + } + } else if (does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NUMBERVALUE); + } + + multipart_startaccum(p); + capture_begin(p, ptr); + return true; +} + +static bool parse_number(upb_json_parser *p, bool is_quoted); + +static bool end_number_nontop(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + return parse_number(p, false); +} + +static bool end_number(upb_json_parser *p, const char *ptr) { + if (!end_number_nontop(p, ptr)) { + return false; + } + + if (does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +/* |buf| is NULL-terminated. |buf| itself will never include quotes; + * |is_quoted| tells us whether this text originally appeared inside quotes. */ +static bool parse_number_from_buffer(upb_json_parser *p, const char *buf, + bool is_quoted) { + size_t len = strlen(buf); + const char *bufend = buf + len; + char *end; + upb_fieldtype_t type = upb_fielddef_type(p->top->f); + double val; + double dummy; + double inf = UPB_INFINITY; + + errno = 0; + + if (len == 0 || buf[0] == ' ') { + return false; + } + + /* For integer types, first try parsing with integer-specific routines. + * If these succeed, they will be more accurate for int64/uint64 than + * strtod(). + */ + switch (type) { + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else if (val > INT32_MAX || val < INT32_MIN) { + return false; + } else { + upb_sink_putint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT32: { + unsigned long val = strtoul(buf, &end, 0); + if (end != bufend) { + break; + } else if (val > UINT32_MAX || errno == ERANGE) { + return false; + } else { + upb_sink_putuint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + /* XXX: We can't handle [u]int64 properly on 32-bit machines because + * strto[u]ll isn't in C89. */ + case UPB_TYPE_INT64: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else { + upb_sink_putint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT64: { + unsigned long val = strtoul(p->accumulated, &end, 0); + if (end != bufend) { + break; + } else if (errno == ERANGE) { + return false; + } else { + upb_sink_putuint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + default: + break; + } + + if (type != UPB_TYPE_DOUBLE && type != UPB_TYPE_FLOAT && is_quoted) { + /* Quoted numbers for integer types are not allowed to be in double form. */ + return false; + } + + if (len == strlen("Infinity") && strcmp(buf, "Infinity") == 0) { + /* C89 does not have an INFINITY macro. */ + val = inf; + } else if (len == strlen("-Infinity") && strcmp(buf, "-Infinity") == 0) { + val = -inf; + } else { + val = strtod(buf, &end); + if (errno == ERANGE || end != bufend) { + return false; + } + } + + switch (type) { +#define CASE(capitaltype, smalltype, ctype, min, max) \ + case UPB_TYPE_ ## capitaltype: { \ + if (modf(val, &dummy) != 0 || val > max || val < min) { \ + return false; \ + } else { \ + upb_sink_put ## smalltype(p->top->sink, parser_getsel(p), \ + (ctype)val); \ + return true; \ + } \ + break; \ + } + case UPB_TYPE_ENUM: + CASE(INT32, int32, int32_t, INT32_MIN, INT32_MAX); + CASE(INT64, int64, int64_t, INT64_MIN, INT64_MAX); + CASE(UINT32, uint32, uint32_t, 0, UINT32_MAX); + CASE(UINT64, uint64, uint64_t, 0, UINT64_MAX); +#undef CASE + + case UPB_TYPE_DOUBLE: + upb_sink_putdouble(p->top->sink, parser_getsel(p), val); + return true; + case UPB_TYPE_FLOAT: + if ((val > FLT_MAX || val < -FLT_MAX) && val != inf && val != -inf) { + return false; + } else { + upb_sink_putfloat(p->top->sink, parser_getsel(p), val); + return true; + } + default: + return false; + } +} + +static bool parse_number(upb_json_parser *p, bool is_quoted) { + size_t len; + const char *buf; + + /* strtol() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + if (!multipart_text(p, "\0", 1, false)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (parse_number_from_buffer(p, buf, is_quoted)) { + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "error parsing number: %s", buf); + multipart_end(p); + return false; + } +} + +static bool parser_putbool(upb_json_parser *p, bool val) { + bool ok; + + if (p->top->f == NULL) { + return true; + } + + if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { + upb_status_seterrf(p->status, + "Boolean value specified for non-bool field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + ok = upb_sink_putbool(p->top->sink, parser_getsel(p), val); + UPB_ASSERT(ok); + + return true; +} + +static bool end_bool(upb_json_parser *p, bool val) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_BOOLVALUE); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_BOOLVALUE)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_BOOLVALUE); + } + + if (p->top->is_unknown_field) { + return true; + } + + if (!parser_putbool(p, val)) { + return false; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static bool end_null(upb_json_parser *p) { + const char *zero_ptr = "0"; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + + /* Fill null_value field. */ + multipart_startaccum(p); + capture_begin(p, zero_ptr); + capture_end(p, zero_ptr + 1); + parse_number(p, false); + + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + + return true; +} + +static bool start_any_stringval(upb_json_parser *p) { + multipart_startaccum(p); + return true; +} + +static bool start_stringval(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_string_wrapper_object(p) || + is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + start_fieldmask_object(p); + return true; + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + start_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRINGVALUE); + } else { + return false; + } + } else if (does_string_wrapper_start(p) || + does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (does_fieldmask_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_fieldmask_object(p); + return true; + } else if (is_wellknown_field(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_field(p, UPB_WELLKNOWN_DURATION)) { + if (!start_subobject(p)) { + return false; + } + start_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_STRINGVALUE); + } + + if (p->top->f == NULL) { + multipart_startaccum(p); + return true; + } + + if (p->top->is_any) { + return start_any_stringval(p); + } + + if (upb_fielddef_isstring(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + if (upb_fielddef_type(p->top->f) == UPB_TYPE_STRING) { + /* For STRING fields we push data directly to the handlers as it is + * parsed. We don't do this yet for BYTES fields, because our base64 + * decoder is not streaming. + * + * TODO(haberman): make base64 decoding streaming also. */ + multipart_start(p, getsel_for_handlertype(p, UPB_HANDLER_STRING)); + return true; + } else { + multipart_startaccum(p); + return true; + } + } else if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL && + upb_fielddef_type(p->top->f) != UPB_TYPE_MESSAGE) { + /* No need to push a frame -- numeric values in quotes remain in the + * current parser frame. These values must accmulate so we can convert + * them all at once at the end. */ + multipart_startaccum(p); + return true; + } else { + upb_status_seterrf(p->status, + "String specified for bool or submessage field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool end_any_stringval(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Set type_url */ + upb_selector_t sel; + upb_jsonparser_frame *inner; + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + multipart_end(p); + + /* Resolve type url */ + if (strncmp(buf, "type.googleapis.com/", 20) == 0 && len > 20) { + const upb_msgdef *payload_type = NULL; + buf += 20; + len -= 20; + + payload_type = upb_symtab_lookupmsg2(p->symtab, buf, len); + if (payload_type == NULL) { + upb_status_seterrf( + p->status, "Cannot find packed type: %.*s\n", (int)len, buf); + return false; + } + + json_parser_any_frame_set_payload_type(p, p->top->any_frame, payload_type); + + return true; + } else { + upb_status_seterrf( + p->status, "Invalid type url: %.*s\n", (int)len, buf); + return false; + } +} + +static bool end_stringval_nontop(upb_json_parser *p) { + bool ok = true; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + multipart_end(p); + return true; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_stringval(p); + } + + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_BYTES: + if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + /* Fall through. */ + + case UPB_TYPE_STRING: { + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + break; + } + + case UPB_TYPE_ENUM: { + /* Resolve enum symbolic name to integer value. */ + const upb_enumdef *enumdef = upb_fielddef_enumsubdef(p->top->f); + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + int32_t int_val = 0; + ok = upb_enumdef_ntoi(enumdef, buf, len, &int_val); + + if (ok) { + upb_selector_t sel = parser_getsel(p); + upb_sink_putint32(p->top->sink, sel, int_val); + } else { + upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf); + } + + break; + } + + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + ok = parse_number(p, true); + break; + + default: + UPB_ASSERT(false); + upb_status_seterrmsg(p->status, "Internal error in JSON decoder"); + ok = false; + break; + } + + multipart_end(p); + + return ok; +} + +static bool end_stringval(upb_json_parser *p) { + /* FieldMask's stringvals have been ended when handling them. Only need to + * close FieldMask here.*/ + if (does_fieldmask_end(p)) { + end_fieldmask_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (!end_stringval_nontop(p)) { + return false; + } + + if (does_string_wrapper_end(p) || + does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION) || + is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + end_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static void start_duration_base(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_duration_base(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char seconds_buf[14]; + char nanos_buf[12]; + char *end; + int64_t seconds = 0; + int32_t nanos = 0; + double val = 0.0; + const char *seconds_membername = "seconds"; + const char *nanos_membername = "nanos"; + size_t fraction_start; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + memset(seconds_buf, 0, 14); + memset(nanos_buf, 0, 12); + + /* Find out base end. The maximus duration is 315576000000, which cannot be + * represented by double without losing precision. Thus, we need to handle + * fraction and base separately. */ + for (fraction_start = 0; fraction_start < len && buf[fraction_start] != '.'; + fraction_start++); + + /* Parse base */ + memcpy(seconds_buf, buf, fraction_start); + seconds = strtol(seconds_buf, &end, 10); + if (errno == ERANGE || end != seconds_buf + fraction_start) { + upb_status_seterrf(p->status, "error parsing duration: %s", + seconds_buf); + return false; + } + + if (seconds > 315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "maximum acceptable value is " + "315576000000"); + return false; + } + + if (seconds < -315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "minimum acceptable value is " + "-315576000000"); + return false; + } + + /* Parse fraction */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf + fraction_start, len - fraction_start); + val = strtod(nanos_buf, &end); + if (errno == ERANGE || end != nanos_buf + len - fraction_start + 1) { + upb_status_seterrf(p->status, "error parsing duration: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + if (seconds < 0) nanos = -nanos; + + /* Clean up buffer */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous arena */ + multipart_startaccum(p); + + return true; +} + +static int parse_timestamp_number(upb_json_parser *p) { + size_t len; + const char *buf; + int val; + + /* atoi() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + multipart_text(p, "\0", 1, false); + + buf = accumulate_getptr(p, &len); + val = atoi(buf); + multipart_end(p); + multipart_startaccum(p); + + return val; +} + +static void start_year(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_year(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_year = parse_timestamp_number(p) - 1900; + return true; +} + +static void start_month(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_month(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mon = parse_timestamp_number(p) - 1; + return true; +} + +static void start_day(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_day(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mday = parse_timestamp_number(p); + return true; +} + +static void start_hour(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_hour(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_hour = parse_timestamp_number(p); + return true; +} + +static void start_minute(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_minute(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_min = parse_timestamp_number(p); + return true; +} + +static void start_second(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_second(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_sec = parse_timestamp_number(p); + return true; +} + +static void start_timestamp_base(upb_json_parser *p) { + memset(&p->tm, 0, sizeof(struct tm)); +} + +static void start_timestamp_fraction(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_timestamp_fraction(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char nanos_buf[12]; + char *end; + double val = 0.0; + int32_t nanos; + const char *nanos_membername = "nanos"; + + memset(nanos_buf, 0, 12); + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (len > 10) { + upb_status_seterrf(p->status, + "error parsing timestamp: at most 9-digit fraction."); + return false; + } + + /* Parse nanos */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf, len); + val = strtod(nanos_buf, &end); + + if (errno == ERANGE || end != nanos_buf + len + 1) { + upb_status_seterrf(p->status, "error parsing timestamp nanos: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + + /* Clean up previous environment */ + multipart_end(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_timestamp_zone(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +#define EPOCH_YEAR 1970 +#define TM_YEAR_BASE 1900 + +static bool isleap(int year) { + return (year % 4) == 0 && (year % 100 != 0 || (year % 400) == 0); +} + +const unsigned short int __mon_yday[2][13] = { + /* Normal years. */ + { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, + /* Leap years. */ + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } +}; + +int64_t epoch(int year, int yday, int hour, int min, int sec) { + int64_t years = year - EPOCH_YEAR; + + int64_t leap_days = years / 4 - years / 100 + years / 400; + + int64_t days = years * 365 + yday + leap_days; + int64_t hours = days * 24 + hour; + int64_t mins = hours * 60 + min; + int64_t secs = mins * 60 + sec; + return secs; +} + + +static int64_t upb_mktime(const struct tm *tp) { + int sec = tp->tm_sec; + int min = tp->tm_min; + int hour = tp->tm_hour; + int mday = tp->tm_mday; + int mon = tp->tm_mon; + int year = tp->tm_year + TM_YEAR_BASE; + + /* Calculate day of year from year, month, and day of month. */ + int mon_yday = ((__mon_yday[isleap(year)][mon]) - 1); + int yday = mon_yday + mday; + + return epoch(year, yday, hour, min, sec); +} + +static bool end_timestamp_zone(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + int hours; + int64_t seconds; + const char *seconds_membername = "seconds"; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (buf[0] != 'Z') { + if (sscanf(buf + 1, "%2d:00", &hours) != 1) { + upb_status_seterrf(p->status, "error parsing timestamp offset"); + return false; + } + + if (buf[0] == '+') { + hours = -hours; + } + + p->tm.tm_hour += hours; + } + + /* Normalize tm */ + seconds = upb_mktime(&p->tm); + + /* Check timestamp boundary */ + if (seconds < -62135596800) { + upb_status_seterrf(p->status, "error parsing timestamp: " + "minimum acceptable value is " + "0001-01-01T00:00:00Z"); + return false; + } + + /* Clean up previous environment */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_fieldmask_path(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + multipart_startaccum(p); + return true; +} + +static bool lower_camel_push( + upb_json_parser *p, upb_selector_t sel, const char *ptr, size_t len) { + const char *limit = ptr + len; + bool first = true; + for (;ptr < limit; ptr++) { + if (*ptr >= 'A' && *ptr <= 'Z' && !first) { + char lower = tolower(*ptr); + upb_sink_putstring(p->top->sink, sel, "_", 1, NULL); + upb_sink_putstring(p->top->sink, sel, &lower, 1, NULL); + } else { + upb_sink_putstring(p->top->sink, sel, ptr, 1, NULL); + } + first = false; + } + return true; +} + +static bool end_fieldmask_path(upb_json_parser *p) { + upb_selector_t sel; + + if (!lower_camel_push( + p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + + multipart_end(p); + return true; +} + +static void start_member(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + multipart_startaccum(p); +} + +/* Helper: invoked during parse_mapentry() to emit the mapentry message's key + * field based on the current contents of the accumulate buffer. */ +static bool parse_mapentry_key(upb_json_parser *p) { + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Emit the key field. We do a bit of ad-hoc parsing here because the + * parser state machine has already decided that this is a string field + * name, and we are reinterpreting it as some arbitrary key type. In + * particular, integer and bool keys are quoted, so we need to parse the + * quoted string contents here. */ + + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_KEY); + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no key"); + return false; + } + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + /* Invoke end_number. The accum buffer has the number's text already. */ + if (!parse_number(p, true)) { + return false; + } + break; + case UPB_TYPE_BOOL: + if (len == 4 && !strncmp(buf, "true", 4)) { + if (!parser_putbool(p, true)) { + return false; + } + } else if (len == 5 && !strncmp(buf, "false", 5)) { + if (!parser_putbool(p, false)) { + return false; + } + } else { + upb_status_seterrmsg(p->status, + "Map bool key not 'true' or 'false'"); + return false; + } + multipart_end(p); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + upb_sink subsink; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, len, &subsink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(subsink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(subsink, sel); + multipart_end(p); + break; + } + default: + upb_status_seterrmsg(p->status, "Invalid field type for map key"); + return false; + } + + return true; +} + +/* Helper: emit one map entry (as a submessage in the map field sequence). This + * is invoked from end_membername(), at the end of the map entry's key string, + * with the map key in the accumulate buffer. It parses the key from that + * buffer, emits the handler calls to start the mapentry submessage (setting up + * its subframe in the process), and sets up state in the subframe so that the + * value parser (invoked next) will emit the mapentry's value field and then + * end the mapentry message. */ + +static bool handle_mapentry(upb_json_parser *p) { + const upb_fielddef *mapfield; + const upb_msgdef *mapentrymsg; + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Map entry: p->top->sink is the seq frame, so we need to start a frame + * for the mapentry itself, and then set |f| in that frame so that the map + * value field is parsed, and also set a flag to end the frame after the + * map-entry value is parsed. */ + if (!check_stack(p)) return false; + + mapfield = p->top->mapfield; + mapentrymsg = upb_fielddef_msgsubdef(mapfield); + + inner = start_jsonparser_frame(p); + p->top->f = mapfield; + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = mapentrymsg; + inner->mapfield = mapfield; + + /* Don't set this to true *yet* -- we reuse parsing handlers below to push + * the key field value to the sink, and these handlers will pop the frame + * if they see is_mapentry (when invoked by the parser state machine, they + * would have just seen the map-entry value, not key). */ + inner->is_mapentry = false; + p->top = inner; + + /* send STARTMSG in submsg frame. */ + upb_sink_startmsg(p->top->sink); + + parse_mapentry_key(p); + + /* Set up the value field to receive the map-entry value. */ + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_VALUE); + p->top->is_mapentry = true; /* set up to pop frame after value is parsed. */ + p->top->mapfield = mapfield; + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no value"); + return false; + } + + return true; +} + +static bool end_membername(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + + if (!p->top->m) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_membername(p); + } else if (p->top->is_map) { + return handle_mapentry(p); + } else { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (upb_strtable_lookup2(p->top->name_table, buf, len, &v)) { + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + + return true; + } else if (p->ignore_json_unknown) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + return false; + } + } +} + +static bool end_any_membername(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (len == 5 && strncmp(buf, "@type", len) == 0) { + upb_strtable_lookup2(p->top->name_table, "type_url", 8, &v); + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + return true; + } else { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } +} + +static void end_member(upb_json_parser *p) { + /* If we just parsed a map-entry value, end that frame too. */ + if (p->top->is_mapentry) { + upb_selector_t sel; + bool ok; + const upb_fielddef *mapfield; + + UPB_ASSERT(p->top > p->stack); + /* send ENDMSG on submsg. */ + upb_sink_endmsg(p->top->sink, p->status); + mapfield = p->top->mapfield; + + /* send ENDSUBMSG in repeated-field-of-mapentries frame. */ + p->top--; + ok = upb_handlers_getselector(mapfield, UPB_HANDLER_ENDSUBMSG, &sel); + UPB_ASSERT(ok); + upb_sink_endsubmsg(p->top->sink, sel); + } + + p->top->f = NULL; + p->top->is_unknown_field = false; +} + +static void start_any_member(upb_json_parser *p, const char *ptr) { + start_member(p); + json_parser_any_frame_set_after_type_url_start_once(p->top->any_frame, ptr); +} + +static void end_any_member(upb_json_parser *p, const char *ptr) { + json_parser_any_frame_set_before_type_url_end(p->top->any_frame, ptr); + end_member(p); +} + +static bool start_subobject(upb_json_parser *p) { + if (p->top->is_unknown_field) { + if (!check_stack(p)) return false; + + p->top = start_jsonparser_frame(p); + return true; + } + + if (upb_fielddef_ismap(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a map. Start a new parser frame in a repeated-field + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->mapfield = p->top->f; + inner->is_map = true; + p->top = inner; + + return true; + } else if (upb_fielddef_issubmsg(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a subobject. Start a new parser frame in the submsg + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + set_name_table(p, inner); + p->top = inner; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + + return true; + } else { + upb_status_seterrf(p->status, + "Object specified for non-message/group field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool start_subobject_full(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + start_structvalue_object(p); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_STRUCT)) { + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } + + return start_subobject(p); +} + +static void end_subobject(upb_json_parser *p) { + if (is_top_level(p)) { + return; + } + + if (p->top->is_map) { + upb_selector_t sel; + p->top--; + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + } else { + upb_selector_t sel; + bool is_unknown = p->top->m == NULL; + p->top--; + if (!is_unknown) { + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(p->top->sink, sel); + } + } +} + +static void end_subobject_full(upb_json_parser *p) { + end_subobject(p); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + end_structvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static bool start_array(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + start_listvalue_object(p); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_LISTVALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } + + if (p->top->is_unknown_field) { + inner = start_jsonparser_frame(p); + inner->is_unknown_field = true; + p->top = inner; + + return true; + } + + if (!upb_fielddef_isseq(p->top->f)) { + upb_status_seterrf(p->status, + "Array specified for non-repeated field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + inner->is_repeated = true; + p->top = inner; + + return true; +} + +static void end_array(upb_json_parser *p) { + upb_selector_t sel; + + UPB_ASSERT(p->top > p->stack); + + p->top--; + + if (p->top->is_unknown_field) { + return; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + end_listvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static void start_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_startmsg(p->top->sink); + } +} + +static void end_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_endmsg(p->top->sink, p->status); + } +} + +static void start_any_object(upb_json_parser *p, const char *ptr) { + start_object(p); + p->top->any_frame->before_type_url_start = ptr; + p->top->any_frame->before_type_url_end = ptr; +} + +static bool end_any_object(upb_json_parser *p, const char *ptr) { + const char *value_membername = "value"; + bool is_well_known_packed = false; + const char *packed_end = ptr + 1; + upb_selector_t sel; + upb_jsonparser_frame *inner; + + if (json_parser_any_frame_has_value(p->top->any_frame) && + !json_parser_any_frame_has_type_url(p->top->any_frame)) { + upb_status_seterrmsg(p->status, "No valid type url"); + return false; + } + + /* Well known types data is represented as value field. */ + if (upb_msgdef_wellknowntype(p->top->any_frame->parser->top->m) != + UPB_WELLKNOWN_UNSPECIFIED) { + is_well_known_packed = true; + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + p->top->any_frame->before_type_url_start = + memchr(p->top->any_frame->before_type_url_start, ':', + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start); + if (p->top->any_frame->before_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "invalid data for well known type."); + return false; + } + p->top->any_frame->before_type_url_start++; + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + p->top->any_frame->after_type_url_start = + memchr(p->top->any_frame->after_type_url_start, ':', + (ptr + 1) - + p->top->any_frame->after_type_url_start); + if (p->top->any_frame->after_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "Invalid data for well known type."); + return false; + } + p->top->any_frame->after_type_url_start++; + packed_end = ptr; + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->before_type_url_start, + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "{", 1, NULL)) { + return false; + } + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame) && + json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, ",", 1, NULL)) { + return false; + } + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->after_type_url_start, + packed_end - p->top->any_frame->after_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "}", 1, NULL)) { + return false; + } + } + } + + if (!end(p->top->any_frame->parser, NULL)) { + return false; + } + + p->top->is_any = false; + + /* Set value */ + start_member(p); + capture_begin(p, value_membername); + capture_end(p, value_membername + 5); + end_membername(p); + + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, p->top->any_frame->stringsink.ptr, + p->top->any_frame->stringsink.len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + end_member(p); + + end_object(p); + + /* Deallocate any parse frame. */ + json_parser_any_frame_free(p->top->any_frame); + + return true; +} + +static bool is_string_wrapper(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_STRINGVALUE || + type == UPB_WELLKNOWN_BYTESVALUE; +} + +static bool is_fieldmask(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_FIELDMASK; +} + +static void start_fieldmask_object(upb_json_parser *p) { + const char *membername = "paths"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); + + start_array(p); +} + +static void end_fieldmask_object(upb_json_parser *p) { + end_array(p); + end_member(p); + end_object(p); +} + +static void start_wrapper_object(upb_json_parser *p) { + const char *membername = "value"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); +} + +static void end_wrapper_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_value_object(upb_json_parser *p, int value_type) { + const char *nullmember = "null_value"; + const char *numbermember = "number_value"; + const char *stringmember = "string_value"; + const char *boolmember = "bool_value"; + const char *structmember = "struct_value"; + const char *listmember = "list_value"; + const char *membername = ""; + + switch (value_type) { + case VALUE_NULLVALUE: + membername = nullmember; + break; + case VALUE_NUMBERVALUE: + membername = numbermember; + break; + case VALUE_STRINGVALUE: + membername = stringmember; + break; + case VALUE_BOOLVALUE: + membername = boolmember; + break; + case VALUE_STRUCTVALUE: + membername = structmember; + break; + case VALUE_LISTVALUE: + membername = listmember; + break; + } + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_value_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_listvalue_object(upb_json_parser *p) { + const char *membername = "values"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_listvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_structvalue_object(upb_json_parser *p) { + const char *membername = "fields"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_structvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static bool is_top_level(upb_json_parser *p) { + return p->top == p->stack && p->top->f == NULL && !p->top->is_unknown_field; +} + +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->m != NULL && upb_msgdef_wellknowntype(p->top->m) == type; +} + +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + (upb_msgdef_wellknowntype(upb_fielddef_msgsubdef(p->top->f)) + == type); +} + +static bool does_number_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + upb_msgdef_isnumberwrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_number_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool is_number_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool does_string_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_string_wrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_string_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool is_string_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool does_fieldmask_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_fieldmask(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_fieldmask_end(upb_json_parser *p) { + return p->top->m != NULL && is_fieldmask(p->top->m); +} + +#define CHECK_RETURN_TOP(x) if (!(x)) goto error + + +/* The actual parser **********************************************************/ + +/* What follows is the Ragel parser itself. The language is specified in Ragel + * and the actions call our C functions above. + * + * Ragel has an extensive set of functionality, and we use only a small part of + * it. There are many action types but we only use a few: + * + * ">" -- transition into a machine + * "%" -- transition out of a machine + * "@" -- transition into a final state of a machine. + * + * "@" transitions are tricky because a machine can transition into a final + * state repeatedly. But in some cases we know this can't happen, for example + * a string which is delimited by a final '"' can only transition into its + * final state once, when the closing '"' is seen. */ + + +#line 2794 "upb/json/parser.rl" + + + +#line 2597 "upb/json/parser.c" +static const char _json_actions[] = { + 0, 1, 0, 1, 1, 1, 3, 1, + 4, 1, 6, 1, 7, 1, 8, 1, + 9, 1, 11, 1, 12, 1, 13, 1, + 14, 1, 15, 1, 16, 1, 17, 1, + 18, 1, 19, 1, 20, 1, 22, 1, + 23, 1, 24, 1, 35, 1, 37, 1, + 39, 1, 40, 1, 42, 1, 43, 1, + 44, 1, 46, 1, 48, 1, 49, 1, + 50, 1, 51, 1, 53, 1, 54, 2, + 4, 9, 2, 5, 6, 2, 7, 3, + 2, 7, 9, 2, 21, 26, 2, 25, + 10, 2, 27, 28, 2, 29, 30, 2, + 32, 34, 2, 33, 31, 2, 38, 36, + 2, 40, 42, 2, 45, 2, 2, 46, + 54, 2, 47, 36, 2, 49, 54, 2, + 50, 54, 2, 51, 54, 2, 52, 41, + 2, 53, 54, 3, 32, 34, 35, 4, + 21, 26, 27, 28 +}; + +static const short _json_key_offsets[] = { + 0, 0, 12, 13, 18, 23, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, + 38, 43, 44, 48, 53, 58, 63, 67, + 71, 74, 77, 79, 83, 87, 89, 91, + 96, 98, 100, 109, 115, 121, 127, 133, + 135, 139, 142, 144, 146, 149, 150, 154, + 156, 158, 160, 162, 163, 165, 167, 168, + 170, 172, 173, 175, 177, 178, 180, 182, + 183, 185, 187, 191, 193, 195, 196, 197, + 198, 199, 201, 206, 208, 210, 212, 221, + 222, 222, 222, 227, 232, 237, 238, 239, + 240, 241, 241, 242, 243, 244, 244, 245, + 246, 247, 247, 252, 253, 257, 262, 267, + 272, 276, 276, 279, 282, 285, 288, 291, + 294, 294, 294, 294, 294, 294 +}; + +static const char _json_trans_keys[] = { + 32, 34, 45, 91, 102, 110, 116, 123, + 9, 13, 48, 57, 34, 32, 93, 125, + 9, 13, 32, 44, 93, 9, 13, 32, + 93, 125, 9, 13, 97, 108, 115, 101, + 117, 108, 108, 114, 117, 101, 32, 34, + 125, 9, 13, 34, 32, 58, 9, 13, + 32, 93, 125, 9, 13, 32, 44, 125, + 9, 13, 32, 44, 125, 9, 13, 32, + 34, 9, 13, 45, 48, 49, 57, 48, + 49, 57, 46, 69, 101, 48, 57, 69, + 101, 48, 57, 43, 45, 48, 57, 48, + 57, 48, 57, 46, 69, 101, 48, 57, + 34, 92, 34, 92, 34, 47, 92, 98, + 102, 110, 114, 116, 117, 48, 57, 65, + 70, 97, 102, 48, 57, 65, 70, 97, + 102, 48, 57, 65, 70, 97, 102, 48, + 57, 65, 70, 97, 102, 34, 92, 45, + 48, 49, 57, 48, 49, 57, 46, 115, + 48, 57, 115, 48, 57, 34, 46, 115, + 48, 57, 48, 57, 48, 57, 48, 57, + 48, 57, 45, 48, 57, 48, 57, 45, + 48, 57, 48, 57, 84, 48, 57, 48, + 57, 58, 48, 57, 48, 57, 58, 48, + 57, 48, 57, 43, 45, 46, 90, 48, + 57, 48, 57, 58, 48, 48, 34, 48, + 57, 43, 45, 90, 48, 57, 34, 44, + 34, 44, 34, 44, 34, 45, 91, 102, + 110, 116, 123, 48, 57, 34, 32, 93, + 125, 9, 13, 32, 44, 93, 9, 13, + 32, 93, 125, 9, 13, 97, 108, 115, + 101, 117, 108, 108, 114, 117, 101, 32, + 34, 125, 9, 13, 34, 32, 58, 9, + 13, 32, 93, 125, 9, 13, 32, 44, + 125, 9, 13, 32, 44, 125, 9, 13, + 32, 34, 9, 13, 32, 9, 13, 32, + 9, 13, 32, 9, 13, 32, 9, 13, + 32, 9, 13, 32, 9, 13, 0 +}; + +static const char _json_single_lengths[] = { + 0, 8, 1, 3, 3, 3, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 3, 1, 2, 3, 3, 3, 2, 2, + 1, 3, 0, 2, 2, 0, 0, 3, + 2, 2, 9, 0, 0, 0, 0, 2, + 2, 1, 2, 0, 1, 1, 2, 0, + 0, 0, 0, 1, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 1, + 0, 0, 4, 0, 0, 1, 1, 1, + 1, 0, 3, 2, 2, 2, 7, 1, + 0, 0, 3, 3, 3, 1, 1, 1, + 1, 0, 1, 1, 1, 0, 1, 1, + 1, 0, 3, 1, 2, 3, 3, 3, + 2, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0 +}; + +static const char _json_range_lengths[] = { + 0, 2, 0, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 3, 0, + 1, 1, 0, 1, 1, 0, 1, 1, + 1, 1, 1, 0, 1, 1, 0, 1, + 1, 0, 1, 1, 0, 1, 1, 0, + 1, 1, 0, 1, 1, 0, 0, 0, + 0, 1, 1, 0, 0, 0, 1, 0, + 0, 0, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 1, 1, 1, + 1, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0 +}; + +static const short _json_index_offsets[] = { + 0, 0, 11, 13, 18, 23, 28, 30, + 32, 34, 36, 38, 40, 42, 44, 46, + 48, 53, 55, 59, 64, 69, 74, 78, + 82, 85, 89, 91, 95, 99, 101, 103, + 108, 111, 114, 124, 128, 132, 136, 140, + 143, 147, 150, 153, 155, 158, 160, 164, + 166, 168, 170, 172, 174, 176, 178, 180, + 182, 184, 186, 188, 190, 192, 194, 196, + 198, 200, 202, 207, 209, 211, 213, 215, + 217, 219, 221, 226, 229, 232, 235, 244, + 246, 247, 248, 253, 258, 263, 265, 267, + 269, 271, 272, 274, 276, 278, 279, 281, + 283, 285, 286, 291, 293, 297, 302, 307, + 312, 316, 317, 320, 323, 326, 329, 332, + 335, 336, 337, 338, 339, 340 +}; + +static const unsigned char _json_indicies[] = { + 0, 2, 3, 4, 5, 6, 7, 8, + 0, 3, 1, 9, 1, 11, 12, 1, + 11, 10, 13, 14, 12, 13, 1, 14, + 1, 1, 14, 10, 15, 1, 16, 1, + 17, 1, 18, 1, 19, 1, 20, 1, + 21, 1, 22, 1, 23, 1, 24, 1, + 25, 26, 27, 25, 1, 28, 1, 29, + 30, 29, 1, 30, 1, 1, 30, 31, + 32, 33, 34, 32, 1, 35, 36, 27, + 35, 1, 36, 26, 36, 1, 37, 38, + 39, 1, 38, 39, 1, 41, 42, 42, + 40, 43, 1, 42, 42, 43, 40, 44, + 44, 45, 1, 45, 1, 45, 40, 41, + 42, 42, 39, 40, 47, 48, 46, 50, + 51, 49, 52, 52, 52, 52, 52, 52, + 52, 52, 53, 1, 54, 54, 54, 1, + 55, 55, 55, 1, 56, 56, 56, 1, + 57, 57, 57, 1, 59, 60, 58, 61, + 62, 63, 1, 64, 65, 1, 66, 67, + 1, 68, 1, 67, 68, 1, 69, 1, + 66, 67, 65, 1, 70, 1, 71, 1, + 72, 1, 73, 1, 74, 1, 75, 1, + 76, 1, 77, 1, 78, 1, 79, 1, + 80, 1, 81, 1, 82, 1, 83, 1, + 84, 1, 85, 1, 86, 1, 87, 1, + 88, 1, 89, 89, 90, 91, 1, 92, + 1, 93, 1, 94, 1, 95, 1, 96, + 1, 97, 1, 98, 1, 99, 99, 100, + 98, 1, 102, 1, 101, 104, 105, 103, + 1, 1, 101, 106, 107, 108, 109, 110, + 111, 112, 107, 1, 113, 1, 114, 115, + 117, 118, 1, 117, 116, 119, 120, 118, + 119, 1, 120, 1, 1, 120, 116, 121, + 1, 122, 1, 123, 1, 124, 1, 125, + 126, 1, 127, 1, 128, 1, 129, 130, + 1, 131, 1, 132, 1, 133, 134, 135, + 136, 134, 1, 137, 1, 138, 139, 138, + 1, 139, 1, 1, 139, 140, 141, 142, + 143, 141, 1, 144, 145, 136, 144, 1, + 145, 135, 145, 1, 146, 147, 147, 1, + 148, 148, 1, 149, 149, 1, 150, 150, + 1, 151, 151, 1, 152, 152, 1, 1, + 1, 1, 1, 1, 1, 0 +}; + +static const char _json_trans_targs[] = { + 1, 0, 2, 107, 3, 6, 10, 13, + 16, 106, 4, 3, 106, 4, 5, 7, + 8, 9, 108, 11, 12, 109, 14, 15, + 110, 16, 17, 111, 18, 18, 19, 20, + 21, 22, 111, 21, 22, 24, 25, 31, + 112, 26, 28, 27, 29, 30, 33, 113, + 34, 33, 113, 34, 32, 35, 36, 37, + 38, 39, 33, 113, 34, 41, 42, 46, + 42, 46, 43, 45, 44, 114, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 73, 72, 68, 69, 70, 71, + 72, 115, 74, 67, 72, 76, 116, 76, + 116, 77, 79, 81, 82, 85, 90, 94, + 98, 80, 117, 117, 83, 82, 80, 83, + 84, 86, 87, 88, 89, 117, 91, 92, + 93, 117, 95, 96, 97, 117, 98, 99, + 105, 100, 100, 101, 102, 103, 104, 105, + 103, 104, 117, 106, 106, 106, 106, 106, + 106 +}; + +static const unsigned char _json_trans_actions[] = { + 0, 0, 113, 107, 53, 0, 0, 0, + 125, 59, 45, 0, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 101, 51, 47, 0, 0, 45, + 49, 49, 104, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 5, 15, + 0, 0, 71, 7, 13, 0, 74, 9, + 9, 9, 77, 80, 11, 37, 37, 37, + 0, 0, 0, 39, 0, 41, 86, 0, + 0, 0, 17, 19, 0, 21, 23, 0, + 25, 27, 0, 29, 31, 0, 33, 35, + 0, 135, 83, 135, 0, 0, 0, 0, + 0, 92, 0, 89, 89, 98, 43, 0, + 131, 95, 113, 107, 53, 0, 0, 0, + 125, 59, 69, 110, 45, 0, 55, 0, + 0, 0, 0, 0, 0, 119, 0, 0, + 0, 122, 0, 0, 0, 116, 0, 101, + 51, 47, 0, 0, 45, 49, 49, 104, + 0, 0, 128, 0, 57, 63, 65, 61, + 67 +}; + +static const unsigned char _json_eof_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 0, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 57, 63, 65, 61, 67, + 0, 0, 0, 0, 0, 0 +}; + +static const int json_start = 1; + +static const int json_en_number_machine = 23; +static const int json_en_string_machine = 32; +static const int json_en_duration_machine = 40; +static const int json_en_timestamp_machine = 47; +static const int json_en_fieldmask_machine = 75; +static const int json_en_value_machine = 78; +static const int json_en_main = 1; + + +#line 2797 "upb/json/parser.rl" + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle) { + upb_json_parser *parser = closure; + + /* Variables used by Ragel's generated code. */ + int cs = parser->current_state; + int *stack = parser->parser_stack; + int top = parser->parser_top; + + const char *p = buf; + const char *pe = buf + size; + const char *eof = &eof_ch; + + parser->handle = handle; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + capture_resume(parser, buf); + + +#line 2875 "upb/json/parser.c" + { + int _klen; + unsigned int _trans; + const char *_acts; + unsigned int _nacts; + const char *_keys; + + if ( p == pe ) + goto _test_eof; + if ( cs == 0 ) + goto _out; +_resume: + _keys = _json_trans_keys + _json_key_offsets[cs]; + _trans = _json_index_offsets[cs]; + + _klen = _json_single_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + _klen - 1; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + ((_upper-_lower) >> 1); + if ( (*p) < *_mid ) + _upper = _mid - 1; + else if ( (*p) > *_mid ) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); + goto _match; + } + } + _keys += _klen; + _trans += _klen; + } + + _klen = _json_range_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( (*p) < _mid[0] ) + _upper = _mid - 2; + else if ( (*p) > _mid[1] ) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys)>>1); + goto _match; + } + } + _trans += _klen; + } + +_match: + _trans = _json_indicies[_trans]; + cs = _json_trans_targs[_trans]; + + if ( _json_trans_actions[_trans] == 0 ) + goto _again; + + _acts = _json_actions + _json_trans_actions[_trans]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) + { + switch ( *_acts++ ) + { + case 1: +#line 2602 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 2: +#line 2604 "upb/json/parser.rl" + { p--; {stack[top++] = cs; cs = 23;goto _again;} } + break; + case 3: +#line 2608 "upb/json/parser.rl" + { start_text(parser, p); } + break; + case 4: +#line 2609 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_text(parser, p)); } + break; + case 5: +#line 2615 "upb/json/parser.rl" + { start_hex(parser); } + break; + case 6: +#line 2616 "upb/json/parser.rl" + { hexdigit(parser, p); } + break; + case 7: +#line 2617 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_hex(parser)); } + break; + case 8: +#line 2623 "upb/json/parser.rl" + { CHECK_RETURN_TOP(escape(parser, p)); } + break; + case 9: +#line 2629 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 10: +#line 2634 "upb/json/parser.rl" + { start_year(parser, p); } + break; + case 11: +#line 2635 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_year(parser, p)); } + break; + case 12: +#line 2639 "upb/json/parser.rl" + { start_month(parser, p); } + break; + case 13: +#line 2640 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_month(parser, p)); } + break; + case 14: +#line 2644 "upb/json/parser.rl" + { start_day(parser, p); } + break; + case 15: +#line 2645 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_day(parser, p)); } + break; + case 16: +#line 2649 "upb/json/parser.rl" + { start_hour(parser, p); } + break; + case 17: +#line 2650 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_hour(parser, p)); } + break; + case 18: +#line 2654 "upb/json/parser.rl" + { start_minute(parser, p); } + break; + case 19: +#line 2655 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_minute(parser, p)); } + break; + case 20: +#line 2659 "upb/json/parser.rl" + { start_second(parser, p); } + break; + case 21: +#line 2660 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_second(parser, p)); } + break; + case 22: +#line 2665 "upb/json/parser.rl" + { start_duration_base(parser, p); } + break; + case 23: +#line 2666 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_duration_base(parser, p)); } + break; + case 24: +#line 2668 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 25: +#line 2673 "upb/json/parser.rl" + { start_timestamp_base(parser); } + break; + case 26: +#line 2675 "upb/json/parser.rl" + { start_timestamp_fraction(parser, p); } + break; + case 27: +#line 2676 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_timestamp_fraction(parser, p)); } + break; + case 28: +#line 2678 "upb/json/parser.rl" + { start_timestamp_zone(parser, p); } + break; + case 29: +#line 2679 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_timestamp_zone(parser, p)); } + break; + case 30: +#line 2681 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 31: +#line 2686 "upb/json/parser.rl" + { start_fieldmask_path_text(parser, p); } + break; + case 32: +#line 2687 "upb/json/parser.rl" + { end_fieldmask_path_text(parser, p); } + break; + case 33: +#line 2692 "upb/json/parser.rl" + { start_fieldmask_path(parser); } + break; + case 34: +#line 2693 "upb/json/parser.rl" + { end_fieldmask_path(parser); } + break; + case 35: +#line 2699 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; + case 36: +#line 2704 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { + {stack[top++] = cs; cs = 47;goto _again;} + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { + {stack[top++] = cs; cs = 40;goto _again;} + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_FIELDMASK)) { + {stack[top++] = cs; cs = 75;goto _again;} + } else { + {stack[top++] = cs; cs = 32;goto _again;} + } + } + break; + case 37: +#line 2717 "upb/json/parser.rl" + { p--; {stack[top++] = cs; cs = 78;goto _again;} } + break; + case 38: +#line 2722 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_member(parser, p); + } else { + start_member(parser); + } + } + break; + case 39: +#line 2729 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_membername(parser)); } + break; + case 40: +#line 2732 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + end_any_member(parser, p); + } else { + end_member(parser); + } + } + break; + case 41: +#line 2743 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_object(parser, p); + } else { + start_object(parser); + } + } + break; + case 42: +#line 2752 "upb/json/parser.rl" + { + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + CHECK_RETURN_TOP(end_any_object(parser, p)); + } else { + end_object(parser); + } + } + break; + case 43: +#line 2764 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_array(parser)); } + break; + case 44: +#line 2768 "upb/json/parser.rl" + { end_array(parser); } + break; + case 45: +#line 2773 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_number(parser, p)); } + break; + case 46: +#line 2774 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_number(parser, p)); } + break; + case 47: +#line 2776 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_stringval(parser)); } + break; + case 48: +#line 2777 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_stringval(parser)); } + break; + case 49: +#line 2779 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, true)); } + break; + case 50: +#line 2781 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, false)); } + break; + case 51: +#line 2783 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_null(parser)); } + break; + case 52: +#line 2785 "upb/json/parser.rl" + { CHECK_RETURN_TOP(start_subobject_full(parser)); } + break; + case 53: +#line 2786 "upb/json/parser.rl" + { end_subobject_full(parser); } + break; + case 54: +#line 2791 "upb/json/parser.rl" + { p--; {cs = stack[--top]; goto _again;} } + break; +#line 3199 "upb/json/parser.c" + } + } + +_again: + if ( cs == 0 ) + goto _out; + if ( ++p != pe ) + goto _resume; + _test_eof: {} + if ( p == eof ) + { + const char *__acts = _json_actions + _json_eof_actions[cs]; + unsigned int __nacts = (unsigned int) *__acts++; + while ( __nacts-- > 0 ) { + switch ( *__acts++ ) { + case 0: +#line 2600 "upb/json/parser.rl" + { p--; {cs = stack[--top]; if ( p == pe ) + goto _test_eof; +goto _again;} } + break; + case 46: +#line 2774 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_number(parser, p)); } + break; + case 49: +#line 2779 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, true)); } + break; + case 50: +#line 2781 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_bool(parser, false)); } + break; + case 51: +#line 2783 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_null(parser)); } + break; + case 53: +#line 2786 "upb/json/parser.rl" + { end_subobject_full(parser); } + break; +#line 3241 "upb/json/parser.c" + } + } + } + + _out: {} + } + +#line 2819 "upb/json/parser.rl" + + if (p != pe) { + upb_status_seterrf(parser->status, "Parse error at '%.*s'\n", pe - p, p); + } else { + capture_suspend(parser, &p); + } + +error: + /* Save parsing state back to parser. */ + parser->current_state = cs; + parser->parser_top = top; + + return p - buf; +} + +static bool end(void *closure, const void *hd) { + upb_json_parser *parser = closure; + + /* Prevent compile warning on unused static constants. */ + UPB_UNUSED(json_start); + UPB_UNUSED(json_en_duration_machine); + UPB_UNUSED(json_en_fieldmask_machine); + UPB_UNUSED(json_en_number_machine); + UPB_UNUSED(json_en_string_machine); + UPB_UNUSED(json_en_timestamp_machine); + UPB_UNUSED(json_en_value_machine); + UPB_UNUSED(json_en_main); + + parse(parser, hd, &eof_ch, 0, NULL); + + return parser->current_state >= 106; +} + +static void json_parser_reset(upb_json_parser *p) { + int cs; + int top; + + p->top = p->stack; + init_frame(p->top); + + /* Emit Ragel initialization of the parser. */ + +#line 3292 "upb/json/parser.c" + { + cs = json_start; + top = 0; + } + +#line 2861 "upb/json/parser.rl" + p->current_state = cs; + p->parser_top = top; + accumulate_clear(p); + p->multipart_state = MULTIPART_INACTIVE; + p->capture = NULL; + p->accumulated = NULL; +} + +static upb_json_parsermethod *parsermethod_new(upb_json_codecache *c, + const upb_msgdef *md) { + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + upb_json_parsermethod *m = upb_malloc(alloc, sizeof(*m)); + + m->cache = c; + + upb_byteshandler_init(&m->input_handler_); + upb_byteshandler_setstring(&m->input_handler_, parse, m); + upb_byteshandler_setendstr(&m->input_handler_, end, m); + + upb_strtable_init2(&m->name_table, UPB_CTYPE_CONSTPTR, alloc); + + /* Build name_table */ + + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_value v = upb_value_constptr(f); + char *buf; + + /* Add an entry for the JSON name. */ + size_t len = upb_fielddef_getjsonname(f, NULL, 0); + buf = upb_malloc(alloc, len); + upb_fielddef_getjsonname(f, buf, len); + upb_strtable_insert3(&m->name_table, buf, strlen(buf), v, alloc); + + if (strcmp(buf, upb_fielddef_name(f)) != 0) { + /* Since the JSON name is different from the regular field name, add an + * entry for the raw name (compliant proto3 JSON parsers must accept + * both). */ + const char *name = upb_fielddef_name(f); + upb_strtable_insert3(&m->name_table, name, strlen(name), v, alloc); + } + } + + return m; +} + +/* Public API *****************************************************************/ + +upb_json_parser *upb_json_parser_create(upb_arena *arena, + const upb_json_parsermethod *method, + const upb_symtab* symtab, + upb_sink output, + upb_status *status, + bool ignore_json_unknown) { +#ifndef NDEBUG + const size_t size_before = upb_arena_bytesallocated(arena); +#endif + upb_json_parser *p = upb_arena_malloc(arena, sizeof(upb_json_parser)); + if (!p) return false; + + p->arena = arena; + p->method = method; + p->status = status; + p->limit = p->stack + UPB_JSON_MAX_DEPTH; + p->accumulate_buf = NULL; + p->accumulate_buf_size = 0; + upb_bytessink_reset(&p->input_, &method->input_handler_, p); + + json_parser_reset(p); + p->top->sink = output; + p->top->m = upb_handlers_msgdef(output.handlers); + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + set_name_table(p, p->top); + p->symtab = symtab; + + p->ignore_json_unknown = ignore_json_unknown; + + /* If this fails, uncomment and increase the value in parser.h. */ + /* fprintf(stderr, "%zd\n", upb_arena_bytesallocated(arena) - size_before); */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= + UPB_JSON_PARSER_SIZE); + return p; +} + +upb_bytessink upb_json_parser_input(upb_json_parser *p) { + return p->input_; +} + +const upb_byteshandler *upb_json_parsermethod_inputhandler( + const upb_json_parsermethod *m) { + return &m->input_handler_; +} + +upb_json_codecache *upb_json_codecache_new(void) { + upb_alloc *alloc; + upb_json_codecache *c; + + c = upb_gmalloc(sizeof(*c)); + + c->arena = upb_arena_new(); + alloc = upb_arena_alloc(c->arena); + + upb_inttable_init2(&c->methods, UPB_CTYPE_CONSTPTR, alloc); + + return c; +} + +void upb_json_codecache_free(upb_json_codecache *c) { + upb_arena_free(c->arena); + upb_gfree(c); +} + +const upb_json_parsermethod *upb_json_codecache_get(upb_json_codecache *c, + const upb_msgdef *md) { + upb_json_parsermethod *m; + upb_value v; + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + if (upb_inttable_lookupptr(&c->methods, md, &v)) { + return upb_value_getconstptr(v); + } + + m = parsermethod_new(c, md); + v = upb_value_constptr(m); + + if (!m) return NULL; + if (!upb_inttable_insertptr2(&c->methods, md, v, alloc)) return NULL; + + /* Populate parser methods for all submessages, so the name tables will + * be available during parsing. */ + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + + if (upb_fielddef_issubmsg(f)) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + const upb_json_parsermethod *sub_method = + upb_json_codecache_get(c, subdef); + + if (!sub_method) return NULL; + } + } + + return m; +} diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh new file mode 100644 index 00000000000..ad8122b1bdb --- /dev/null +++ b/kokoro/ubuntu/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Install the latest version of Bazel. +use_bazel.sh latest + +# Verify/query CMake +echo PATH=$PATH +ls -l `which cmake` +cmake --version + +# Log the bazel path and version. +which bazel +bazel version + +cd $(dirname $0)/../.. +bazel test --test_output=errors :all diff --git a/kokoro/ubuntu/continuous.cfg b/kokoro/ubuntu/continuous.cfg new file mode 100644 index 00000000000..fa97583b342 --- /dev/null +++ b/kokoro/ubuntu/continuous.cfg @@ -0,0 +1,2 @@ +build_file: "upb/kokoro/ubuntu/build.sh" +timeout_mins: 15 diff --git a/kokoro/ubuntu/presubmit.cfg b/kokoro/ubuntu/presubmit.cfg new file mode 100644 index 00000000000..fa97583b342 --- /dev/null +++ b/kokoro/ubuntu/presubmit.cfg @@ -0,0 +1,2 @@ +build_file: "upb/kokoro/ubuntu/build.sh" +timeout_mins: 15 diff --git a/tests/benchmark.cc b/tests/benchmark.cc new file mode 100644 index 00000000000..bcb4ec78b63 --- /dev/null +++ b/tests/benchmark.cc @@ -0,0 +1,36 @@ + +#include +#include +#include "google/protobuf/descriptor.upb.h" +#include "google/protobuf/descriptor.upbdefs.h" + +upb_strview descriptor = google_protobuf_descriptor_proto_upbdefinit.descriptor; + +/* A buffer big enough to parse descriptor.proto without going to heap. */ +char buf[65535]; + +static void BM_CreateArena(benchmark::State& state) { + for (auto _ : state) { + upb_arena* arena = upb_arena_init(buf, sizeof(buf), NULL); + upb_arena_free(arena); + } +} +BENCHMARK(BM_CreateArena); + +static void BM_ParseDescriptor(benchmark::State& state) { + size_t bytes = 0; + for (auto _ : state) { + upb_arena* arena = upb_arena_init(buf, sizeof(buf), NULL); + google_protobuf_FileDescriptorProto* set = + google_protobuf_FileDescriptorProto_parse(descriptor.data, + descriptor.size, arena); + if (!set) { + printf("Failed to parse.\n"); + exit(1); + } + bytes += descriptor.size; + upb_arena_free(arena); + } + state.SetBytesProcessed(state.iterations() * descriptor.size); +} +BENCHMARK(BM_ParseDescriptor); diff --git a/tests/bindings/googlepb/test_vs_proto2.cc b/tests/bindings/googlepb/test_vs_proto2.cc new file mode 100644 index 00000000000..ac447e1b0cb --- /dev/null +++ b/tests/bindings/googlepb/test_vs_proto2.cc @@ -0,0 +1,165 @@ +/* + * + * A test that verifies that our results are identical to proto2 for a + * given proto type and input protobuf. + */ + +#define __STDC_LIMIT_MACROS // So we get UINT32_MAX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tests/google_messages.pb.h" +#include "tests/upb_test.h" +#include "upb/bindings/googlepb/bridge.h" +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/pb/glue.h" +#include "upb/pb/varint.int.h" + +// Pull in string data from tests/google_message{1,2}.dat +// (the .h files are generated with xxd). +const unsigned char message1_data[] = { +#include "tests/google_message1.h" +}; + +const unsigned char message2_data[] = { +#include "tests/google_message2.h" +}; + +void compare_metadata(const google::protobuf::Descriptor* d, + const upb::MessageDef *upb_md) { + ASSERT(d->field_count() == upb_md->field_count()); + for (upb::MessageDef::const_field_iterator i = upb_md->field_begin(); + i != upb_md->field_end(); ++i) { + const upb::FieldDef* upb_f = *i; + const google::protobuf::FieldDescriptor *proto2_f = + d->FindFieldByNumber(upb_f->number()); + ASSERT(upb_f); + ASSERT(proto2_f); + ASSERT(upb_f->number() == (uint32_t)proto2_f->number()); + ASSERT(std::string(upb_f->name()) == proto2_f->name()); + ASSERT(upb_f->descriptor_type() == + static_cast(proto2_f->type())); + ASSERT(upb_f->IsSequence() == proto2_f->is_repeated()); + } +} + +void print_diff(const google::protobuf::Message& msg1, + const google::protobuf::Message& msg2) { + std::string text_str1; + std::string text_str2; + google::protobuf::TextFormat::PrintToString(msg1, &text_str1); + google::protobuf::TextFormat::PrintToString(msg2, &text_str2); + fprintf(stderr, "str1: %s, str2: %s\n", text_str1.c_str(), text_str2.c_str()); +} + +void parse_and_compare(google::protobuf::Message *msg1, + google::protobuf::Message *msg2, + const upb::Handlers *protomsg_handlers, + const char *str, size_t len, bool allow_jit) { + // Parse to both proto2 and upb. + ASSERT(msg1->ParseFromArray(str, len)); + + upb::pb::CodeCache cache; + ASSERT(cache.set_allow_jit(allow_jit)); + upb::reffed_ptr decoder_method( + cache.GetDecoderMethod(upb::pb::DecoderMethodOptions(protomsg_handlers))); + + upb::Status status; + upb::Environment env; + env.ReportErrorsTo(&status); + upb::Sink protomsg_sink(protomsg_handlers, msg2); + upb::pb::Decoder* decoder = + upb::pb::Decoder::Create(&env, decoder_method.get(), &protomsg_sink); + + msg2->Clear(); + bool ok = upb::BufferSource::PutBuffer(str, len, decoder->input()); + if (!ok) { + fprintf(stderr, "error parsing: %s\n", status.error_message()); + print_diff(*msg1, *msg2); + } + ASSERT(ok); + ASSERT(status.ok()); + + // Would like to just compare the message objects themselves, but + // unfortunately MessageDifferencer is not part of the open-source release of + // proto2, so we compare their serialized strings, which we expect will be + // equivalent. + std::string str1; + std::string str2; + msg1->SerializeToString(&str1); + msg2->SerializeToString(&str2); + if (str1 != str2) { + print_diff(*msg1, *msg2); + } + ASSERT(str1 == str2); + ASSERT(std::string(str, len) == str2); +} + +void test_zig_zag() { + for (uint64_t num = 5; num * 1.5 < UINT64_MAX; num *= 1.5) { + ASSERT(upb_zzenc_64(num) == + google::protobuf::internal::WireFormatLite::ZigZagEncode64(num)); + if (num < UINT32_MAX) { + ASSERT(upb_zzenc_32(num) == + google::protobuf::internal::WireFormatLite::ZigZagEncode32(num)); + } + } + +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + UPB_UNUSED(message1_data); + UPB_UNUSED(message2_data); + size_t len = sizeof(MESSAGE_DATA_IDENT); + const char *str = (const char*)MESSAGE_DATA_IDENT; + + MESSAGE_CIDENT msg1; + MESSAGE_CIDENT msg2; + + upb::reffed_ptr h( + upb::googlepb::WriteHandlers::New(msg1)); + + compare_metadata(msg1.GetDescriptor(), h->message_def()); + + // Run twice to test proper object reuse. + parse_and_compare(&msg1, &msg2, h.get(), str, len, false); + parse_and_compare(&msg1, &msg2, h.get(), str, len, true); + parse_and_compare(&msg1, &msg2, h.get(), str, len, false); + parse_and_compare(&msg1, &msg2, h.get(), str, len, true); + + // Test with DynamicMessage. + google::protobuf::DynamicMessageFactory* factory = + new google::protobuf::DynamicMessageFactory; + const google::protobuf::Message* prototype = + factory->GetPrototype(msg1.descriptor()); + google::protobuf::Message* dyn_msg1 = prototype->New(); + google::protobuf::Message* dyn_msg2 = prototype->New(); + h = upb::googlepb::WriteHandlers::New(*dyn_msg1); + parse_and_compare(dyn_msg1, dyn_msg2, h.get(), str, len, false); + parse_and_compare(dyn_msg1, dyn_msg2, h.get(), str, len, true); + delete dyn_msg1; + delete dyn_msg2; + delete factory; + + test_zig_zag(); + + printf("All tests passed, %d assertions.\n", num_assertions); + + google::protobuf::ShutdownProtobufLibrary(); + return 0; +} + +} diff --git a/tests/bindings/lua/test_upb.lua b/tests/bindings/lua/test_upb.lua new file mode 100644 index 00000000000..e4edda40032 --- /dev/null +++ b/tests/bindings/lua/test_upb.lua @@ -0,0 +1,750 @@ + +local upb = require "upb" +local lunit = require "lunit" + +if _VERSION >= 'Lua 5.2' then + _ENV = lunit.module("testupb", "seeall") +else + module("testupb", lunit.testcase, package.seeall) +end + +function iter_to_array(iter) + local arr = {} + for v in iter do + arr[#arr + 1] = v + end + return arr +end + +function test_msgdef() + local f2 = upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32} + local o = upb.OneofDef{name = "field1", fields = {f2}} + local f = upb.FieldDef{name = "field3", number = 2, type = upb.TYPE_INT32} + + local m = upb.MessageDef{fields = {o, f}} + + assert_equal(f, m:lookup_name("field3")) + assert_equal(o, m:lookup_name("field1")) + assert_equal(f2, m:lookup_name("field2")) +end + +function test_fielddef() + local f = upb.FieldDef() + assert_false(f:is_frozen()) + assert_nil(f:number()) + assert_nil(f:name()) + assert_nil(f:type()) + assert_equal(upb.LABEL_OPTIONAL, f:label()) + + f:set_name("foo_field") + f:set_number(3) + f:set_label(upb.LABEL_REPEATED) + f:set_type(upb.TYPE_FLOAT) + + assert_equal("foo_field", f:name()) + assert_equal(3, f:number()) + assert_equal(upb.LABEL_REPEATED, f:label()) + assert_equal(upb.TYPE_FLOAT, f:type()) + + local f2 = upb.FieldDef{ + name = "foo", number = 5, type = upb.TYPE_DOUBLE, label = upb.LABEL_REQUIRED + } + + assert_equal("foo", f2:name()) + assert_equal(5, f2:number()) + assert_equal(upb.TYPE_DOUBLE, f2:type()) + assert_equal(upb.LABEL_REQUIRED, f2:label()) +end + +function test_enumdef() + local e = upb.EnumDef() + assert_equal(0, #e) + assert_nil(e:value(5)) + assert_nil(e:value("NONEXISTENT_NAME")) + + for name, value in e:values() do + fail() + end + + e:add("VAL1", 1) + e:add("VAL2", 2) + + local values = {} + for name, value in e:values() do + values[name] = value + end + + assert_equal(1, values["VAL1"]) + assert_equal(2, values["VAL2"]) + + local e2 = upb.EnumDef{ + values = { + {"FOO", 1}, + {"BAR", 77}, + } + } + + assert_equal(1, e2:value("FOO")) + assert_equal(77, e2:value("BAR")) + assert_equal("FOO", e2:value(1)) + assert_equal("BAR", e2:value(77)) + + e2:freeze() + + local f = upb.FieldDef{type = upb.TYPE_ENUM} + + -- No default set and no EnumDef to get a default from. + assert_equal(f:default(), nil) + + f:set_subdef(upb.EnumDef()) + -- No default to pull in from the EnumDef. + assert_equal(f:default(), nil) + + f:set_subdef(e2) + -- First member added to e2. + assert_equal(f:default(), "FOO") + + f:set_subdef(nil) + assert_equal(f:default(), nil) + + f:set_default(1) + assert_equal(f:default(), 1) + + f:set_default("YOYOYO") + assert_equal(f:default(), "YOYOYO") + + f:set_subdef(e2) + f:set_default(1) + -- It prefers to return a string, and could resolve the explicit "1" we set + -- it to to the string value. + assert_equal(f:default(), "FOO") + + -- FieldDef can specify default value by name or number, but the value must + -- exist at freeze time. + local m1 = upb.build_defs{ + upb.MessageDef{ + full_name = "A", + fields = { + upb.FieldDef{ + name = "f1", + number = 1, + type = upb.TYPE_ENUM, + subdef = e2, + default = "BAR" + }, + upb.FieldDef{ + name = "f2", + number = 2, + type = upb.TYPE_ENUM, + subdef = e2, + default = 77 + } + } + } + } + + assert_equal(m1:field("f1"):default(), "BAR") + assert_equal(m1:field("f1"):default(), "BAR") + + assert_error_match( + "enum default for field A.f1 .DOESNT_EXIST. is not in the enum", + function() + local m1 = upb.build_defs{ + upb.MessageDef{ + full_name = "A", + fields = { + upb.FieldDef{ + name = "f1", + number = 1, + type = upb.TYPE_ENUM, + subdef = e2, + default = "DOESNT_EXIST" + } + } + } + } + end + ) + + assert_error_match( + "enum default for field A.f1 .142. is not in the enum", + function() + local m1 = upb.build_defs{ + upb.MessageDef{ + full_name = "A", + fields = { + upb.FieldDef{ + name = "f1", + number = 1, + type = upb.TYPE_ENUM, + subdef = e2, + default = 142 + } + } + } + } + end + ) +end + +function test_empty_msgdef() + local md = upb.MessageDef() + assert_nil(md:full_name()) -- Def without name is anonymous. + assert_false(md:is_frozen()) + assert_equal(0, #md) + assert_nil(md:field("nonexistent_field")) + assert_nil(md:field(3)) + for field in md:fields() do + fail() + end + + upb.freeze(md) + assert_true(md:is_frozen()) + assert_equal(0, #md) + assert_nil(md:field("nonexistent_field")) + assert_nil(md:field(3)) + for field in md:fields() do + fail() + end +end + +function test_msgdef_constructor() + local f1 = upb.FieldDef{name = "field1", number = 7, type = upb.TYPE_INT32} + local f2 = upb.FieldDef{name = "field2", number = 8, type = upb.TYPE_INT32} + local md = upb.MessageDef{ + full_name = "TestMessage", + fields = {f1, f2} + } + assert_equal("TestMessage", md:full_name()) + assert_false(md:is_frozen()) + assert_equal(2, #md) + assert_equal(f1, md:field("field1")) + assert_equal(f2, md:field("field2")) + assert_equal(f1, md:field(7)) + assert_equal(f2, md:field(8)) + local count = 0 + local found = {} + for field in md:fields() do + count = count + 1 + found[field] = true + end + assert_equal(2, count) + assert_true(found[f1]) + assert_true(found[f2]) + + upb.freeze(md) +end + +function test_iteration() + -- Test that we cannot crash the process even if we modify the set of fields + -- during iteration. + local md = upb.MessageDef{full_name = "TestMessage"} + + for i=1,10 do + md:add(upb.FieldDef{ + name = "field" .. tostring(i), + number = 1000 - i, + type = upb.TYPE_INT32 + }) + end + + local add = #md + for f in md:fields() do + if add > 0 then + add = add - 1 + for i=10000,11000 do + local field_name = "field" .. tostring(i) + -- We want to add fields to the table to trigger a table resize, + -- but we must skip it if the field name or number already exists + -- otherwise it will raise an error. + if md:field(field_name) == nil and + md:field(i) == nil then + md:add(upb.FieldDef{ + name = field_name, + number = i, + type = upb.TYPE_INT32 + }) + end + end + end + end + + -- Test that iterators don't crash the process even if the MessageDef goes + -- out of scope. + -- + -- Note: have previously verified that this can indeed crash the process if + -- we do not explicitly add a reference from the iterator to the underlying + -- MessageDef. + local iter = md:fields() + md = nil + collectgarbage() + while iter() do + end + + local ed = upb.EnumDef{ + values = { + {"FOO", 1}, + {"BAR", 77}, + } + } + iter = ed:values() + ed = nil + collectgarbage() + while iter() do + end +end + +function test_msgdef_setters() + local md = upb.MessageDef() + md:set_full_name("Message1") + assert_equal("Message1", md:full_name()) + local f = upb.FieldDef{name = "field1", number = 3, type = upb.TYPE_DOUBLE} + md:add(f) + assert_equal(1, #md) + assert_equal(f, md:field("field1")) +end + +function test_msgdef_errors() + assert_error(function() upb.MessageDef{bad_initializer_key = 5} end) + local md = upb.MessageDef() + assert_error(function() + -- Duplicate field number. + upb.MessageDef{ + fields = { + upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32} + } + } + end) + assert_error(function() + -- Duplicate field name. + upb.MessageDef{ + fields = { + upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "field1", number = 2, type = upb.TYPE_INT32} + } + } + end) + + assert_error(function() + -- Duplicate field name. + upb.MessageDef{ + fields = { + upb.OneofDef{name = "field1", fields = { + upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32}, + }}, + upb.FieldDef{name = "field2", number = 2, type = upb.TYPE_INT32} + } + } + end) + + -- attempt to set a name with embedded NULLs. + assert_error_match("names cannot have embedded NULLs", function() + md:set_full_name("abc\0def") + end) + + upb.freeze(md) + -- Attempt to mutate frozen MessageDef. + assert_error_match("frozen", function() + md:add(upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}) + end) + assert_error_match("frozen", function() + md:set_full_name("abc") + end) + + -- Attempt to freeze a msgdef without freezing its subdef. + assert_error_match("is not frozen or being frozen", function() + m1 = upb.MessageDef() + upb.freeze( + upb.MessageDef{ + fields = { + upb.FieldDef{name = "f1", number = 1, type = upb.TYPE_MESSAGE, + subdef = m1} + } + } + ) + end) +end + +function test_symtab() + local empty = upb.SymbolTable() + assert_equal(0, #iter_to_array(empty:defs(upb.DEF_ANY))) + assert_equal(0, #iter_to_array(empty:defs(upb.DEF_MSG))) + assert_equal(0, #iter_to_array(empty:defs(upb.DEF_ENUM))) + + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage"}, + upb.MessageDef{full_name = "ContainingMessage", fields = { + upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "field2", number = 2, type = upb.TYPE_MESSAGE, + subdef_name = ".TestMessage"} + } + } + } + + local msgdef1 = symtab:lookup("TestMessage") + local msgdef2 = symtab:lookup("ContainingMessage") + assert_not_nil(msgdef1) + assert_not_nil(msgdef2) + assert_equal(msgdef1, msgdef2:field("field2"):subdef()) + assert_true(msgdef1:is_frozen()) + assert_true(msgdef2:is_frozen()) + + symtab:add{ + upb.MessageDef{full_name = "ContainingMessage2", fields = { + upb.FieldDef{name = "field5", number = 5, type = upb.TYPE_MESSAGE, + subdef = msgdef2} + } + } + } + + local msgdef3 = symtab:lookup("ContainingMessage2") + assert_not_nil(msgdef3) + assert_equal(msgdef3:field("field5"):subdef(), msgdef2) +end + +function test_numeric_array() + local function test_for_numeric_type(upb_type, val, too_big, too_small, bad3) + local array = upb.Array(upb_type) + assert_equal(0, #array) + + -- 0 is never a valid index in Lua. + assert_error_match("array index", function() return array[0] end) + -- Past the end of the array. + assert_error_match("array index", function() return array[1] end) + + array[1] = val + assert_equal(val, array[1]) + assert_equal(1, #array) + assert_equal(val, array[1]) + -- Past the end of the array. + assert_error_match("array index", function() return array[2] end) + + array[2] = 10 + assert_equal(val, array[1]) + assert_equal(10, array[2]) + assert_equal(2, #array) + -- Past the end of the array. + assert_error_match("array index", function() return array[3] end) + + local n = 1 + for i, val in upb.ipairs(array) do + assert_equal(n, i) + n = n + 1 + assert_equal(array[i], val) + end + + -- Values that are out of range. + local errmsg = "not an integer or out of range" + if too_small then + assert_error_match(errmsg, function() array[3] = too_small end) + end + if too_big then + assert_error_match(errmsg, function() array[3] = too_big end) + end + if bad3 then + assert_error_match(errmsg, function() array[3] = bad3 end) + end + + -- Can't assign other Lua types. + errmsg = "bad argument #3" + assert_error_match(errmsg, function() array[3] = "abc" end) + assert_error_match(errmsg, function() array[3] = true end) + assert_error_match(errmsg, function() array[3] = false end) + assert_error_match(errmsg, function() array[3] = nil end) + assert_error_match(errmsg, function() array[3] = {} end) + assert_error_match(errmsg, function() array[3] = print end) + assert_error_match(errmsg, function() array[3] = array end) + end + + -- in-range of 64-bit types but not exactly representable as double + local bad64 = 2^68 - 1 + + test_for_numeric_type(upb.TYPE_UINT32, 2^32 - 1, 2^32, -1, 5.1) + test_for_numeric_type(upb.TYPE_UINT64, 2^63, 2^64, -1, bad64) + test_for_numeric_type(upb.TYPE_INT32, 2^31 - 1, 2^31, -2^31 - 1, 5.1) + -- Enums don't exist at a language level in Lua, so we just represent enum + -- values as int32s. + test_for_numeric_type(upb.TYPE_ENUM, 2^31 - 1, 2^31, -2^31 - 1, 5.1) + test_for_numeric_type(upb.TYPE_INT64, 2^62, 2^63, -2^64, bad64) + test_for_numeric_type(upb.TYPE_FLOAT, 340282306073709652508363335590014353408) + test_for_numeric_type(upb.TYPE_DOUBLE, 10^101) +end + +function test_string_array() + local function test_for_string_type(upb_type) + local array = upb.Array(upb_type) + assert_equal(0, #array) + + -- 0 is never a valid index in Lua. + assert_error_match("array index", function() return array[0] end) + -- Past the end of the array. + assert_error_match("array index", function() return array[1] end) + + array[1] = "foo" + assert_equal("foo", array[1]) + assert_equal(1, #array) + -- Past the end of the array. + assert_error_match("array index", function() return array[2] end) + + local array2 = upb.Array(upb_type) + assert_equal(0, #array2) + + array[2] = "bar" + assert_equal("foo", array[1]) + assert_equal("bar", array[2]) + assert_equal(2, #array) + -- Past the end of the array. + assert_error_match("array index", function() return array[3] end) + + local n = 1 + for i, val in upb.ipairs(array) do + assert_equal(n, i) + n = n + 1 + assert_equal(array[i], val) + end + assert_equal(3, n) + + -- Can't assign other Lua types. + assert_error_match("Expected string", function() array[3] = 123 end) + assert_error_match("Expected string", function() array[3] = true end) + assert_error_match("Expected string", function() array[3] = false end) + assert_error_match("Expected string", function() array[3] = nil end) + assert_error_match("Expected string", function() array[3] = {} end) + assert_error_match("Expected string", function() array[3] = print end) + assert_error_match("Expected string", function() array[3] = array end) + end + + test_for_string_type(upb.TYPE_STRING) + test_for_string_type(upb.TYPE_BYTES) +end + +function test_msg_primitives() + local function test_for_numeric_type(upb_type, val, too_big, too_small, bad3) + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "f", number = 1, type = upb_type}, + } + } + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + msg = TestMessage() + + -- Defaults to zero + assert_equal(0, msg.f) + + msg.f = 0 + assert_equal(0, msg.f) + + msg.f = val + assert_equal(val, msg.f) + + local errmsg = "not an integer or out of range" + if too_small then + assert_error_match(errmsg, function() msg.f = too_small end) + end + if too_big then + assert_error_match(errmsg, function() msg.f = too_big end) + end + if bad3 then + assert_error_match(errmsg, function() msg.f = bad3 end) + end + + -- Can't assign other Lua types. + errmsg = "bad argument #3" + assert_error_match(errmsg, function() msg.f = "abc" end) + assert_error_match(errmsg, function() msg.f = true end) + assert_error_match(errmsg, function() msg.f = false end) + assert_error_match(errmsg, function() msg.f = nil end) + assert_error_match(errmsg, function() msg.f = {} end) + assert_error_match(errmsg, function() msg.f = print end) + assert_error_match(errmsg, function() msg.f = array end) + end + + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{ + name = "i32", number = 1, type = upb.TYPE_INT32, default = 1}, + upb.FieldDef{ + name = "u32", number = 2, type = upb.TYPE_UINT32, default = 2}, + upb.FieldDef{ + name = "i64", number = 3, type = upb.TYPE_INT64, default = 3}, + upb.FieldDef{ + name = "u64", number = 4, type = upb.TYPE_UINT64, default = 4}, + upb.FieldDef{ + name = "dbl", number = 5, type = upb.TYPE_DOUBLE, default = 5}, + upb.FieldDef{ + name = "flt", number = 6, type = upb.TYPE_FLOAT, default = 6}, + upb.FieldDef{ + name = "bool", number = 7, type = upb.TYPE_BOOL, default = true}, + } + } + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + msg = TestMessage() + + -- Unset member returns default value. + -- TODO(haberman): re-enable these when we have descriptor-based reflection. + -- assert_equal(1, msg.i32) + -- assert_equal(2, msg.u32) + -- assert_equal(3, msg.i64) + -- assert_equal(4, msg.u64) + -- assert_equal(5, msg.dbl) + -- assert_equal(6, msg.flt) + -- assert_equal(true, msg.bool) + + -- Attempts to access non-existent fields fail. + assert_error_match("no such field", function() msg.no_such = 1 end) + + msg.i32 = 10 + msg.u32 = 20 + msg.i64 = 30 + msg.u64 = 40 + msg.dbl = 50 + msg.flt = 60 + msg.bool = true + + assert_equal(10, msg.i32) + assert_equal(20, msg.u32) + assert_equal(30, msg.i64) + assert_equal(40, msg.u64) + assert_equal(50, msg.dbl) + assert_equal(60, msg.flt) + assert_equal(true, msg.bool) + + test_for_numeric_type(upb.TYPE_UINT32, 2^32 - 1, 2^32, -1, 5.1) + test_for_numeric_type(upb.TYPE_UINT64, 2^62, 2^64, -1, bad64) + test_for_numeric_type(upb.TYPE_INT32, 2^31 - 1, 2^31, -2^31 - 1, 5.1) + test_for_numeric_type(upb.TYPE_INT64, 2^61, 2^63, -2^64, bad64) + test_for_numeric_type(upb.TYPE_FLOAT, 2^20) + test_for_numeric_type(upb.TYPE_DOUBLE, 10^101) +end + +function test_msg_array() + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "i32_array", number = 1, type = upb.TYPE_INT32, + label = upb.LABEL_REPEATED}, + } + } + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + msg = TestMessage() + + assert_nil(msg.i32_array) + + -- Can't assign a scalar; array is expected. + assert_error_match("lupb.array expected", function() msg.i32_array = 5 end) + + -- Can't assign array of the wrong type. + local function assign_int64() + msg.i32_array = upb.Array(upb.TYPE_INT64) + end + assert_error_match("Array had incorrect type", assign_int64) + + local arr = upb.Array(upb.TYPE_INT32) + msg.i32_array = arr + assert_equal(arr, msg.i32_array) + + -- Can't assign other Lua types. + assert_error_match("array expected", function() msg.i32_array = "abc" end) + assert_error_match("array expected", function() msg.i32_array = true end) + assert_error_match("array expected", function() msg.i32_array = false end) + assert_error_match("array expected", function() msg.i32_array = nil end) + assert_error_match("array expected", function() msg.i32_array = {} end) + assert_error_match("array expected", function() msg.i32_array = print end) +end + +function test_msg_submsg() + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "submsg", number = 1, type = upb.TYPE_MESSAGE, + subdef_name = ".SubMessage"}, + } + }, + upb.MessageDef{full_name = "SubMessage"} + } + + factory = upb.MessageFactory(symtab) + TestMessage = factory:get_message_class("TestMessage") + SubMessage = factory:get_message_class("SubMessage") + msg = TestMessage() + + assert_nil(msg.submsg) + + -- Can't assign message of the wrong type. + local function assign_int64() + msg.submsg = TestMessage() + end + assert_error_match("Message had incorrect type", assign_int64) + + local sub = SubMessage() + msg.submsg = sub + assert_equal(sub, msg.submsg) + + -- Can't assign other Lua types. + assert_error_match("msg expected", function() msg.submsg = "abc" end) + assert_error_match("msg expected", function() msg.submsg = true end) + assert_error_match("msg expected", function() msg.submsg = false end) + assert_error_match("msg expected", function() msg.submsg = nil end) + assert_error_match("msg expected", function() msg.submsg = {} end) + assert_error_match("msg expected", function() msg.submsg = print end) +end + +-- Lua 5.1 and 5.2 have slightly different semantics for how a finalizer +-- can be defined in Lua. +if _VERSION >= 'Lua 5.2' then + function defer(fn) + setmetatable({}, { __gc = fn }) + end +else + function defer(fn) + getmetatable(newproxy(true)).__gc = fn + end +end + +function test_finalizer() + -- Tests that we correctly handle a call into an already-finalized object. + -- Collectible objects are finalized in the opposite order of creation. + do + local t = {} + defer(function() + assert_error_match("called into dead object", function() + -- Generic def call. + t[1]:full_name() + end) + assert_error_match("called into dead object", function() + -- Specific msgdef call. + t[1]:add() + end) + assert_error_match("called into dead object", function() + t[2]:values() + end) + assert_error_match("called into dead object", function() + t[3]:number() + end) + assert_error_match("called into dead object", function() + t[4]:lookup() + end) + end) + t = { + upb.MessageDef(), + upb.EnumDef(), + upb.FieldDef(), + upb.SymbolTable(), + } + end + collectgarbage() +end + +local stats = lunit.main() + +if stats.failed > 0 or stats.errors > 0 then + error("One or more errors in test suite") +end diff --git a/tests/bindings/lua/test_upb.pb.lua b/tests/bindings/lua/test_upb.pb.lua new file mode 100644 index 00000000000..ea6de099895 --- /dev/null +++ b/tests/bindings/lua/test_upb.pb.lua @@ -0,0 +1,80 @@ + +-- Require "pb" first to ensure that the transitive require of "upb" is +-- handled properly by the "pb" module. +local pb = require "upb.pb" +local upb = require "upb" +local lunit = require "lunit" + +if _VERSION >= 'Lua 5.2' then + _ENV = lunit.module("testupb_pb", "seeall") +else + module("testupb_pb", lunit.testcase, package.seeall) +end + +local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "i32", number = 1, type = upb.TYPE_INT32}, + upb.FieldDef{name = "u32", number = 2, type = upb.TYPE_UINT32}, + upb.FieldDef{name = "i64", number = 3, type = upb.TYPE_INT64}, + upb.FieldDef{name = "u64", number = 4, type = upb.TYPE_UINT64}, + upb.FieldDef{name = "dbl", number = 5, type = upb.TYPE_DOUBLE}, + upb.FieldDef{name = "flt", number = 6, type = upb.TYPE_FLOAT}, + upb.FieldDef{name = "bool", number = 7, type = upb.TYPE_BOOL}, + } + } +} + +local factory = upb.MessageFactory(symtab); +local TestMessage = factory:get_message_class("TestMessage") + +function test_parse_primitive() + local binary_pb = + "\008\128\128\128\128\002\016\128\128\128\128\004\024\128\128" + .. "\128\128\128\128\128\002\032\128\128\128\128\128\128\128\001\041\000" + .. "\000\000\000\000\000\248\063\053\000\000\096\064\056\001" + local msg = TestMessage() + pb.decode(msg, binary_pb) + assert_equal(536870912, msg.i32) + assert_equal(1073741824, msg.u32) + assert_equal(1125899906842624, msg.i64) + assert_equal(562949953421312, msg.u64) + assert_equal(1.5, msg.dbl) + assert_equal(3.5, msg.flt) + assert_equal(true, msg.bool) + + local encoded = pb.encode(msg) + local msg2 = TestMessage() + pb.decode(msg2, encoded) + assert_equal(536870912, msg.i32) + assert_equal(1073741824, msg.u32) + assert_equal(1125899906842624, msg.i64) + assert_equal(562949953421312, msg.u64) + assert_equal(1.5, msg.dbl) + assert_equal(3.5, msg.flt) + assert_equal(true, msg.bool) +end + +function test_parse_string() + local symtab = upb.SymbolTable{ + upb.MessageDef{full_name = "TestMessage", fields = { + upb.FieldDef{name = "str", number = 1, type = upb.TYPE_STRING}, + } + } + } + + local factory = upb.MessageFactory(symtab); + local TestMessage = factory:get_message_class("TestMessage") + + local binary_pb = "\010\005Hello" + msg = TestMessage() + pb.decode(msg, binary_pb) + -- TODO(haberman): re-enable when this stuff works better. + -- assert_equal("Hello", msg.str) +end + + +local stats = lunit.main() + +if stats.failed > 0 or stats.errors > 0 then + error("One or more errors in test suite") +end diff --git a/tests/bindings/ruby/upb.rb b/tests/bindings/ruby/upb.rb new file mode 100644 index 00000000000..3e06c1798c6 --- /dev/null +++ b/tests/bindings/ruby/upb.rb @@ -0,0 +1,62 @@ +#!/usr/bin/ruby +# +# Tests for Ruby upb extension. + +require 'test/unit' +require 'set' +require 'upb' + +def get_descriptor + File.open("upb/descriptor/descriptor.pb").read +end + +def load_descriptor + symtab = Upb::SymbolTable.new + symtab.load_descriptor(get_descriptor()) + return symtab +end + +def get_message_class(name) + return Upb.get_message_class(load_descriptor().lookup(name)) +end + +class TestRubyExtension < Test::Unit::TestCase + def test_parsedescriptor + msgdef = load_descriptor.lookup("google.protobuf.FileDescriptorSet") + assert_instance_of(Upb::MessageDef, msgdef) + + file_descriptor_set = Upb.get_message_class(msgdef) + msg = file_descriptor_set.parse(get_descriptor()) + + # A couple message types we know should exist. + names = Set.new(["DescriptorProto", "FieldDescriptorProto"]) + + msg.file.each { |file| + file.message_type.each { |message_type| + names.delete(message_type.name) + } + } + + assert_equal(0, names.size) + end + + def test_parseserialize + field_descriptor_proto = get_message_class("google.protobuf.FieldDescriptorProto") + field_options = get_message_class("google.protobuf.FieldOptions") + + field = field_descriptor_proto.new + + field.name = "MyName" + field.number = 5 + field.options = field_options.new + field.options.packed = true + + serialized = Upb::Message.serialize(field) + + field2 = field_descriptor_proto.parse(serialized) + + assert_equal("MyName", field2.name) + assert_equal(5, field2.number) + assert_equal(true, field2.options.packed) + end +end diff --git a/tests/conformance_upb.c b/tests/conformance_upb.c new file mode 100644 index 00000000000..6063c9941ed --- /dev/null +++ b/tests/conformance_upb.c @@ -0,0 +1,179 @@ +/* This is a upb implementation of the upb conformance tests, see: + * https://github.com/google/protobuf/tree/master/conformance + */ + +#include +#include +#include +#include +#include + +#include "conformance/conformance.upb.h" +#include "src/google/protobuf/test_messages_proto3.upb.h" + +int test_count = 0; + +bool CheckedRead(int fd, void *buf, size_t len) { + size_t ofs = 0; + while (len > 0) { + ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + + if (bytes_read == 0) return false; + + if (bytes_read < 0) { + perror("reading from test runner"); + exit(1); + } + + len -= bytes_read; + ofs += bytes_read; + } + + return true; +} + +void CheckedWrite(int fd, const void *buf, size_t len) { + if ((size_t)write(fd, buf, len) != len) { + perror("writing to test runner"); + exit(1); + } +} + +bool strview_eql(upb_strview view, const char *str) { + return view.size == strlen(str) && memcmp(view.data, str, view.size) == 0; +} + +static const char *proto3_msg = + "protobuf_test_messages.proto3.TestAllTypesProto3"; + +void DoTest( + const conformance_ConformanceRequest* request, + conformance_ConformanceResponse *response, + upb_arena *arena) { + protobuf_test_messages_proto3_TestAllTypesProto3 *test_message; + + if (!strview_eql(conformance_ConformanceRequest_message_type(request), + proto3_msg)) { + static const char msg[] = "Only proto3 for now."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + + switch (conformance_ConformanceRequest_payload_case(request)) { + case conformance_ConformanceRequest_payload_protobuf_payload: { + upb_strview payload = conformance_ConformanceRequest_protobuf_payload(request); + test_message = protobuf_test_messages_proto3_TestAllTypesProto3_parse( + payload.data, payload.size, arena); + + if (!test_message) { + static const char msg[] = "Parse error"; + conformance_ConformanceResponse_set_parse_error( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + break; + } + + case conformance_ConformanceRequest_payload_NOT_SET: + fprintf(stderr, "conformance_upb: Request didn't have payload.\n"); + return; + + default: { + static const char msg[] = "Unsupported input format."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + } + + switch (conformance_ConformanceRequest_requested_output_format(request)) { + case conformance_UNSPECIFIED: + fprintf(stderr, "conformance_upb: Unspecified output format.\n"); + exit(1); + + case conformance_PROTOBUF: { + size_t serialized_len; + char *serialized = + protobuf_test_messages_proto3_TestAllTypesProto3_serialize( + test_message, arena, &serialized_len); + if (!serialized) { + static const char msg[] = "Error serializing."; + conformance_ConformanceResponse_set_serialize_error( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + conformance_ConformanceResponse_set_protobuf_payload( + response, upb_strview_make(serialized, serialized_len)); + break; + } + + default: { + static const char msg[] = "Unsupported output format."; + conformance_ConformanceResponse_set_skipped( + response, upb_strview_make(msg, sizeof(msg))); + return; + } + } + + return; +} + +bool DoTestIo(void) { + upb_arena *arena; + upb_alloc *alloc; + upb_status status; + char *serialized_input; + char *serialized_output; + uint32_t input_size; + size_t output_size; + conformance_ConformanceRequest *request; + conformance_ConformanceResponse *response; + + if (!CheckedRead(STDIN_FILENO, &input_size, sizeof(uint32_t))) { + /* EOF. */ + return false; + } + + arena = upb_arena_new(); + alloc = upb_arena_alloc(arena); + serialized_input = upb_malloc(alloc, input_size); + + if (!CheckedRead(STDIN_FILENO, serialized_input, input_size)) { + fprintf(stderr, "conformance_upb: unexpected EOF on stdin.\n"); + exit(1); + } + + request = + conformance_ConformanceRequest_parse(serialized_input, input_size, arena); + response = conformance_ConformanceResponse_new(arena); + + if (request) { + DoTest(request, response, arena); + } else { + fprintf(stderr, "conformance_upb: parse of ConformanceRequest failed: %s\n", + upb_status_errmsg(&status)); + } + + serialized_output = conformance_ConformanceResponse_serialize( + response, arena, &output_size); + + CheckedWrite(STDOUT_FILENO, &output_size, sizeof(uint32_t)); + CheckedWrite(STDOUT_FILENO, serialized_output, output_size); + + test_count++; + + upb_arena_free(arena); + + return true; +} + +int main(void) { + while (1) { + if (!DoTestIo()) { + fprintf(stderr, "conformance_upb: received EOF from test runner " + "after %d tests, exiting\n", test_count); + return 0; + } + } +} diff --git a/tests/conformance_upb_failures.txt b/tests/conformance_upb_failures.txt new file mode 100644 index 00000000000..05aab585e77 --- /dev/null +++ b/tests/conformance_upb_failures.txt @@ -0,0 +1 @@ +Required.ProtobufInput.PrematureEofInSubmessageValue.MESSAGE diff --git a/tests/corpus/README b/tests/corpus/README new file mode 100644 index 00000000000..9bd8f1e93d6 --- /dev/null +++ b/tests/corpus/README @@ -0,0 +1 @@ +Corpus folder for fuzzing diff --git a/tests/corpus/temp.cc b/tests/corpus/temp.cc new file mode 100644 index 00000000000..2bf1160ce69 --- /dev/null +++ b/tests/corpus/temp.cc @@ -0,0 +1 @@ +// Hello World diff --git a/tests/file_descriptor_parsenew_fuzzer.cc b/tests/file_descriptor_parsenew_fuzzer.cc new file mode 100644 index 00000000000..057e62d8bf8 --- /dev/null +++ b/tests/file_descriptor_parsenew_fuzzer.cc @@ -0,0 +1,15 @@ +#include + +#include "google/protobuf/descriptor.upb.h" +#include "upb/upb.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + upb::Arena arena; + google_protobuf_FileDescriptorProto_parse(reinterpret_cast(data), + size, arena.ptr()); + return 0; +} + +#ifndef HAVE_FUZZER +int main() {} +#endif diff --git a/tests/google_message1.dat b/tests/google_message1.dat new file mode 100644 index 0000000000000000000000000000000000000000..bc0f064cc2ebd108e4a8ee14d94c59d2207f0f19 GIT binary patch literal 228 zcmd;L5a5tF-Cw7~X6&tPrm5@|X=tEnRGJxFnjhg~7^hiQo0;t9U>Bf~WmBSDS5g&| zlWuPxtfpa;QDp8KR;->=t!|p&o}ZB%uVGuFkQ%H|TpjOepXpql=pO5%t`$&fs-adM zZ)H+l7nQ)6!BD_h70Dqfpi_4tW~D( zryZr6A6c3h9#v%GY;BsERHIew5ajHXR9aRPXq>K*t>d?nG3389$2}=7^$m<_I~W-| c7#Supa#h;~mZzoGDd$zsVCk5cxPVO_0C9jt)c^nh literal 0 HcmV?d00001 diff --git a/tests/google_message2.dat b/tests/google_message2.dat new file mode 100644 index 0000000000000000000000000000000000000000..06c09441b9e1a3fc8577f6c36dbb583d85481b2c GIT binary patch literal 84570 zcmXtfhhtOq`nLlviu7K`bsJ66W;A;?J148zZDzAil4kF{N3G%jT(}o1AS$v&MO09> zC@Q!RaG|0CqM}?y5#jxw^!NS&-{(Age4giA`oKlAo=Sao;L*0|hu0~a?)zb@m}!O`=VpYwL~nlsbYCoB5lzUr4+*9MBDHos1J{owC6zP3?s$qfGY zJ`{|7u=u}kV-GypE_5D>aub)_b?w{*&kHZO^$sEFuYHGtM$)NAAKrD-ii;nu;}7iJ#3 z`n-=%JKn2&wEoA|l~>#|cYd32*@ek-cHr;Ok3N5FGw#Ta=x^DlFC3cj?^UZ$KK^Iy z)i2i4u7C2}+Et=eg^O<7^~1^S^DQqtaota)6SMhh=J>T8={`?nkxEr<}F6|#C zy|!C~;XLU)BV6tNQL^`@7c)l_ubxcIh%Y+ykz{`M zInRen_kOYR#oWP%7JFtB9$K*ET;tZSQ6JwjUw`nt_ywk|uNBW;{opxErmyeJ=^yKx z4PB(6iIvZ97yr{geE)lE{Y338U!c$4vFMK*_Fs16#ml!`TYUWz z{*3DC&uiJ@6?e<#eR=w+;B&?ko8KkB_QP3L;GS4){K3P=csG+_g(qRjPqU&a!HGa z^q&dYE1!Pwlg~U(rSbjr-?yDzA>8y!^zr-G&O7$Qf9@E0xJKT7XT2Mx@zgqyWYQW#ooowe*3~{>xug6O7P8#En^g}V-NkDJ*Ect0{Ii3 z3YJOO_3-h%g938ppXKLDzdHJsZSzBQL1*s=j#T@^#bYRumHx2m)u`j5>z;Y-Jlv?z zaeYhw=$Fe*F8_V|W9gOZYpy?7Sp9F!vwc6M=6N63^3jSJblfKo&YTTD$2(^F->#$n zz4I1eUw(@6??x7;+f3oZn5*v--(I-oGWgONd%cdUFgH?exe)il$@uwK!o-#K3tbjCdC?|NJG z*S~jPsM>hLj*Gv>UBf^6;!#iQ2AZq%*1M{C^BXsR{ncN~?3Y$p=ia~T<0meMLZ>;8 z5sD93MaECHH{7-TRm!tU+Ii&sx1UPayMM5|UxRN?YQ}!ITrzUk8~)Ba_9f!|Ptx4q zOL%u$wjO>}l=|TW>gXf=-N?`w>+5Rvl33r-%TWJ*nH;R?Sg%iGC0v(a_oe~|H{A3_ zC9#h%ZRb0!qc5L%=6Wx4=Z5m=uUi7=|8!uF^`U2HP^kC$A04p@N;f5B*K|^Af7|iQ z;>DTlJ*O{^{5Zm{vOoT2?w-pwZTYr3``q{D-lV$iX2O?u96vw0Y14V3TaLt!NoR2~ z{1cBp$(pvAdP8dAzgl~~NZ`hfDE4%BUVI5GJ=OQn*3I_IXWe&r?uy&#dxst9zkhh~ zqBA>=V1Ir5b@JRhKi=`y4RfCR>V@c5!;K4;hAunBD;>YQHt(1J>^|^hxM_*|Pw(5D zShr!7On>t)n{Fh@Hhr;Z_WDO_e(|RluYXrI`eW%^^iz8lEmyX-uHJRxG1~lR9{cjv zeebQ>erxlV`Oi-K=A6GD>VEFOBYy+Sf8p;(m!}pzKY1BzeCK6LxZbNB*Im1y_VudW zFv`Nbn)If%we{5l2d+B1=d1<&-O_0`;x+IK^Oj0Omov&~qQ&U7YX+-wCr3O~3Zy8u z#^lb$@4b2Cfuprk=l-Yfiiaj3)XIl1hK9r6)gil$=zza@GXsTC6o#=asPX80SWmSn zxF)5_X-Gx*h7J#(8fr`~eCUDa=dWE2%#c@zyEZ* zG;~Fa>ea~x`dwNpFDcP_nJO+-Q$sf<4<7jG%{hC2I&KLtO5Mow&%Ca zt0o`3r$_LL*WZ<*)?Fo>BG`f763v1_c&x2UA}d_uu8h)6pxe0~>}Z`6SEK7i-{>e> z#%a4916538a_O!ub3QxpfErU!V55Q z_RplKT}Pyno@r;mmpp$36yi0so@h7&DMlKQHx$Df3j&5WpX|>VwOviYj~(m^gDrm5 z%^Xwm;AnQlGpuza2efg(*ZdRj?6~F8|01MUU$j$-qBTrY>K%Y@*ldA9)Vd{G9fCDD z0;d(|;tf_6!QWJ1v;^LO-C`Nj`y@(VQ5!eWads)&+nBs({^1Y)n%sOh(B1UDPV)XqA0CA+c48!GYc)1VfP>t z*naQbe=c16`=*}59SCigqJFyBHKmd1aO&zEfI^&?C*;fFSRNwMDM6UTO_~)Y4Wo^% zGq{|_6=m$!@D;A=GTa%&EZE!0+K&AVrvA0(02qqvD+EcEBr-cS=;WCk`v} zqwql4A|A96$VLiqn>$zE{@l-hzP7%nwEH#tkx+HO%fH$x4KeciU{$B|@LGYeU6DXX zWVM!yZxcy0E_2x<(?+D+8Y|cs5a@gkpK2m3Z%nR#W5vTSocQ*&oYl=a-w*xn{6`F>PtMZ0Oo_>j*Jq$eJqhCJx z_OXB5h*>@g+z6pUclR8M1#@k0BktfZ!^%=1T_1(adR#j!(b;4%bpKE?((!|2GJpO0 zw|?KW_4OVv|9F8zGF%sk0WWA+^1ua9XdobglQ|YwFY$=1B^56?Jf^|HajT&0uVT!G zl9XRl!9dkt?^w0tu5}llI(J6j1JnLb%M>^ReJ=^oMdRAmP|>M&6YwrprJhn%7-Dv_ z7;6h!6K+qTg=@<3B#AFA@l;jmD6X;apEq7Uc=Y7***$f?zwsADH(B_+xpXLWHLSx` z<+)(DU3I0qT#}V%Ed=mgb_dp;_u*R_5Ml;9Q6xT>DVqpaB9yFgxS7Zz|P$pMBG>)=hg{I$$ZJ00;zBCgHdvXOG za~M;^CzDPxp_Bs3Sath5kDq*PPmd*;%>tobfG%6b@<3TzxPNjRSkJkrXvVB24k&mz}lERw&1thzx)1PCL-V94X;R1!DVcO z#Ch;v%Px_Is4S+p4f27O%@uRnfwo41(dQv_DQO7_2%2JIHDl>6RLa2J*_DV?S>{bLfvt z5jVT~@u8{on}=_DaT*k2cP30^;hs}G*|<*%D{Ncvp- z0~tb5lKd5G zfV1g+-BSicyJq+g^K$Ss6K%U1?zGrIiIG@C7J>pZ(pD33s(Qh!=7W5)XxB@d?mDmp zJe}3I>yC#csHZLuP37(Y+&H%hg|1O%JQ1HBpALqE&OxWbPml)ck$^Ku90XZy_tTHu z{?`KFCA0eOa$F-pbt4REuO-RCZ>cYYLf7d8G(k+9H$!6OILz1Ns@{pQT8}n5$E2vkE2xO*e%L+Y0vW7Ea;h?6E-^8b8-{9`lBI%o*hsc) zw=wzkp-11H`|SI`4`%f-e*Rd3GClX}R4e6yu`k^x4P6r(!;vFIx-6e(mlP~qxLVNG z)ozA3JqAkqPtU%&?dNr2#4kTv_uf?c8-XwH`$Q5#R|i^zyf8&f%I!ljh{yCcWNbkq zRi>)q5M7dRg3x;R^xr>!e+Y$qeC07AgyIUHoT9;jCB3 z3f&Gd5{=0<%XZv->ql#=$X9nS$3v*ni?*qVnt<l#K$gtP~<}r#%h!HS5fl3${asAtyzuvs-&YmHjJwTPBW~vt< zfpH-`^ZB?mq+>`j))Gw0*W;Zsx|@nog0`rNE)AxQ=zdc_kDbz^*)fW*j`7R+Vo7Iw zu+te$$n)q4a#ZajKoxU>Xzw~ECI%XlfBtXwsh!U~g=ER;eV6R%wI@D!XKNG+$#Kfj zQE9`9i|cgm364k26*i@YxQ(L?TRUAn)kk7CV!XPPn#skb{^o>2<;=Wg4*k)qa5yKTxpAh`I_ZJ-q366Tg)sIzeg zC^c6$uEE+>G*BIh=yp;XXs~p4Y%V<>RdT`Lw0g&$hflAyA?mtX*Bc@I@TZ-*k`PLW z=0cA0P*^Kh%9xr~$V@7U3*&OfL@Oc!9q`QyzIp$ygZEyJaPimdb0O5BmskIPF7$^? zATgk0`dB(NiebZ}Zb2##u?F=VCk)HHbdb6qUA^weyMHy1*bLA57la})=OW5X!JA*4 zE(uALL3gW-QxIJ8Agy7R=}l%`CtwwE<-|dP=_g_mxyiz^M#^J^E4nArcS6eJvtO!r8n6({z zxf*E6X`NhW!WJ)Tn}c?aMTW8XMh2Q12x$lw?|k#NhqjysW}De}-N{d+sB`wLMHoZD zOSZS65S=+}!iF8zqSn$4GD>8uOjPsX6ar>V9rKyd4wj`fWOrLk%?yo5HYt*zBiOob z-mOo)1m@+Ledk^5l@8BxJb(~1!_O+uk%pprcRfpykR|zwT-!|3#9oIsn{MZ7LfW{^ z$I^0@hIp-|(TLS7QY{@|gxSVAT*^WtDYUKYaRhuu%D`wpPtK-M)UHB}?CEqNx)7u- zvVhVoYlAtu!!h4=P+jHgfk31qp^@pCCst?oPk6o8Qp({lms>;X5HMRXt z3@KhAi>t+mG6B;2zSB=X@!oe&9qoPgi~ESkXVLJTM$igft80d(Wn)@Rmv_kpnM$ml z!1KfQ(uCRs;^)3EZ+&|1uPYw`4`=jUZkaxnbN@D8ZX6khana4?Y9J|GF~EiXVV* z?S5f}ttjAk-^kS0ZwYV9zQr`d#T{ z47Kp`+<3QT0ZG+2%BY8!0VYt^f;Gp_?7wL?5ee2){C`8JF9aD-8g@<_04h3le4dY{tPsR?wfP(|b&t~7?J*5R>XTMok??LdxFgrSVW4NR*>A0Lin zC?aiw%c%o{E`0xoS0DLp!}~oBuf&NV)V&5jQnY8mBs#!?_?EC%H0a6GNj(41go@=I zDotcDl{TI|sAvktd_kLPls8gxB`YwtKVjkKA~|IVC$a*F=kW2j)_#23n*l`aJ5}ou zwaTU3RB+fWwk;$X?j)iaY`ez9O8r(9-qd_< zvIsL7_~9j9fGg#y_VJ+*yTBo9v_z9`(56$@@MVccEB6YGep!d-wr9q4X1l@DvNuR^ zBFKktuQ+k!okNcSlGFQ|6Sp8f_iy-#TmJ!Yi=~-rx~PIOp{}y0^o-G_*EU!$8uazV z2#A*XTX!Eh{>W(rOF3DD)y8x>QZ3Gq3$rG8tvEw z>D-7@73a#EL9r@p!n<+|IYB70lTG=cy-8EFjX`EVD~3^q8#-Q{+L%1>>E7jEY<&4j z#JfZ0KL7;z#dA}oNCUtB_)}n@@ff>ift+Z^2Q(?EtWK3;hY9qqLSB%t<$e~I?P^S( z`FhdHU+%dZgy!_Vd(K&j*zFwn#1Sy+VuFc0Y!vrf;JSl@rHf@FG~b{v(56MJpsKTk zXaLH5{M_;%9@%&RffA+d@nlP~Dmw$uFT{}K5&9R5Q6mNgMM5=_U1DH&(KBjd^ z4X{J(u=J;J1`s+2=kNaJrrZANS$)Gc4TQSmd=o-O2w(FT$b{EgU4pztMpTJ0YOkSe zrU)YDcnp`vnahO6rUeIIx$V$RZ}b}0?gW@upZ`WarQGY`1w$b6BttZ*$VuoZ1Og$w z;;h-AAWj_4jU+fyAHTn8tI`Ra2rD0iZNRhb6_1!<9wgwJd3C@};Dg9M{QbTSJJ#(0 zsMGYm+_TRj>KTOpng>#*R5g)NFy*ERO#+gSbD*${Xe}7?TwWI76B>37N5j!8*~SFj zO@NuEfi4uc4|>xhe0y@tP6@?86*v-~l~~GBi}a7P)q%XW!B^^?89X^9wQwYs z`~=1?HiOFj)BfGFe||rUh(7JM7D9alevw{-LXfBo$p%beJbkdjY_jy40)J4;7rR;P z5G$sD94()Q9x_GS760h9c*Prvo#+Q+&GE2GPL zSQ;7~wx|HYsTz`o$NULa$f&O&J4;0h`Lv5Lcn(pR<<^Djx#>EFfr@46HosU0w^EZIZjJ}&r zo&?JhK!XZ@$~Yekk&b4NA#&J;T_#mH!OHS7FwTMIjS>f}9D7FyUcdFp%G)=tfDsj0 zMhp<@Iu~Qxv3HcOPHY^6utl56}#X6?odzDE1td@Lfv!& z9jV$b*s~j8mkfNT%??JDIZP)m!;cW~gXnH`RLl%D!d_K|=@pZ$YPGvI~W&$`3~1bS;?@2s)%LHYd|_Y=nT3l~5;?jdF50 zQRewgjma(Zx9qw5mU|J`oZfe-YH}*wT)*fBkZ!zQ$R`+~5p3-dsX8zcFy;OI)_iTW z%pQs&CE}0VDK?;sVg(!3Z+s(7KYm#juxue zf_dPq`*tk*{H>Qi1`N;YlllGwp$3k=K7|-u3XqO8$ln8;fH20Obq0J5fgl>~O1*HN zV@&cy5g7n0K0JB)*?Tws(0ea5d88=%n*CFq@OAJXq7--!cGkcUBiSK(TI~@u2ZKXK zrzGUmX}ef3$89+L;l6LTrjXAYK3@f(`Ky#ue#(HyBpfKjmCI_x3?F7-I${tX`r(k3 znew^0L+m)A2^&Z3nh?K%#gw{PwJ0hyI3-Spdg86)Q}ut#XE!2l5r7{) z^NS?J3YKwS6+FHLu#xZo z`0l+I)~&t)Vf4VVeh5|n!#b67Oz_DHm%uJ;56Z4+E7*qg>>@( zGdGrSj2FxbYpqQHA+vGe%3FV&JvBerZ<&r{mVS8NvOXw82qmixg0X{9DVRA$G+B!% zI9igEPbrL-Gu5zHPK-EJZW6$^K7am&cRqP<9u1M^+7*{WsGo@cS9S;CzjqNpZ#q5! zfKItlQqYksNy4y;K?+23F_@|nd-%n?Vz@CvA1C+=3fjPs$XLqvhZ7o~*_CU`{fT&t zqelZyCl|bQ$FE;4`m7h8GrtDQP1D!+BW_!PLpXqt^2}y|*qJgZ0v3bQKB$dUWPXlJ zTP1p%ZZ10^w3E^Ns)CK_*G+VX#5BL$DbMkI2A5%!H@oENJ-YO45o}^6YPZ@Okg z1W>U>Osld;FAye%Lu4}!O{-&S7=LS+EeCzz!6)X;`}3y{5QJt{pVs=g6eYi+HRS=+HzMm8{AaKM zX1*@3JT^ASoUqcpap#zU=NlO>=8~cQyttZ#^1Xh{{FMCu0b3%c;kywRBc{w z?#IIZB}{-RtIEZ)KL|8pS* z62fW!r(noc4~f*JxF_Q>Q!D9iM-db2b3q#$XR?9Xa&XzYSD)Csc2`f!*I&qlP~Gni zP4&bPc#C5-=-zXBBA;U%04m3x zT=L0*KNc+m^k(#(`_!dVxg`z%oI5KGG1a1yqOJ-PoOrB+R_40#gW*8}M;jgE=O7;K=Bk%czD%tVbaTayD9)u+E0wHv5u}pCuidtP!`>HrM*t@_ zb|BQ35A-0TG<>}WxU-z-^;_c_VT9xu!D}M?R8X7N`fPb3q3y#POt1o%aCXL$SxKra z$X6w)A-C5(25T+0ltZ6RaRhEKAN=&t=?{*q`}*Gq+ve1dBEYs4{+`_jf!lpvnHe%uw^-T@3!U7+`R}u|1jHFZ{CgwjM{%Chn4=j&qkH-dKkp@|8hML~6x> zK!c%W=R+x9&P>dWM*?P}jU`veCklL4GZgh(36R_B%ep6sHCq?7@jFkid4AEM+j}<& zOW>3gwRY|m2tyon zIn2rXUz@w(DLf+7d-JLY4FUX)wFE_m{4&_x5ngtj#M1t9j zE}fL<2P4gnvj=~D>9h(_64nwyXjR|^ukZoNWtQ2nX@XtXX_rSVtYMnI5GEDdahj<@ zCi;q9W*D%t;;|Kg`Bk1u=s*pGYP={0TKo7fPl^MVm!Km^aYm<87AgYAN3Adk;yv@|7FAVODL zMaY%yRv|Pd>}`N8)tnzbz4?vxZ#>v@QHg#+iu!!f7gNRKhKvxbeWB58*kX?wGlXOz zI9`f3sZLS1*&WEr*=2v1>`IRX)D&u_pJ*oXxpZs;hXGrVdAHrX?fo_L|AVOH-M`%k zM_0i!c1xiUOJz=07_ve(*J9#G7=tGR*;pgRXx1+kWPJ%t%#{^Z6g*#OXj~v3Al2$_ za0>G7TVJpHY{Q-FdKSM`panxc*!lPJD;AzVA6#>wQ>~!_zMp|_>J6gU7~ei#Xs2A- zq)jbn^PnNEyRm-flMlbR^?#tw&FI_l00c3R*^mt{eEm9c2%Gk@EG)ST_A&>kC8Du^ zv?*tf>%#d)G>j?6x`rZHoi07PYSnum{_$Hc98qtLBT5>FA6txg@i2i=O)=<$MrzxE z&&0V>MNQ!}!p?M=hp;YL)BbA>x0dcP>8lfdq5zLRyw8-~G_96G(ubIBJJbpZ;W?N;-B})d7_G-4ag6 z@tFF9R|DjA&07!t zaPR9Y!0prYzF&qZ2)qAKN-t_1rf6v*u`Fwy-Gw{uIyJfWm=yUwsoblXQh4K{H>4q%uG{aT2L@GCak$2-azg?)WrSFk zj#OE}mJwT*bj9I{I9X}gjE-QN01xum)UpjD9P^KbI$dfFAk~vM&)xgmJv-O*d~9K4 z7b0B^cFrq-vkfLRJQ`s%y%L`}XLp7K<$i!6(YT;=K7VNI)~$=51l`BrHpv^Sa zuCdzDW6Ezlkz}(ixrG&7l6)0nx6QQI>1!~2jjjz2n z_h@e%sJ+$;Ee8Bg+6o+)uhW+$iUI~t>W)N*t zIHeZfVF18j>3s{1EWYn+W>X4x7xS1UElSk+wMjhgEiQX@U0^)p-zP#^X@@XpEIOFr-;(zUMv^e)acT$X;m{*fIJc6mI_6sf>CxY+mFBbq5{6G4La{kp$7;R%QCT_N*6c z$K*mrC79E1IkD>ddmlNuwx{zyCx4fsUN^slaCJVs^dLBU#6v_Zn;-91ZQ5W-HJJCBROTzzs6I34Y2n8xhJ@h(U z=$8%A(*5FUzsNGI6;v3`y1HI77la{(A3HE;HH^}fkq*fewd4{u4l#`HQX7*SwyfFn z)%~|1r*<>?&Ro!)5)Ov_vx~s1ET)#hs}dHQ30w^jWRGY9!-SX^?=d(Bsstfu>6dmP$MT`$=R?C zgK10p`<0YJJR&l)V#Pp5O;L3xWLQij+-~5}xfI9~XV$#)>o2!|gj|4t?SZU!7kMo_ zyb$1=SE*r1LnU8J~=(;(E6`;i;3ozq~z z+6C}S6dYKtR1AaYQKBnlEon`IF=yS)^AN#&Ss3h+jTYi~f!>BHjmhV>et!Jb#}3`v z(@yc0_ob*G-&l`Gcp+^3@hbpL4-qx4%$OERkhtWI!*1o*%xwx^&u~Cg&3Ktjv-}dG)h-J&kMBQ>S?ZOZ= zqD(2xHTI9OA+t*BY*~2Nh$J=y*)5ug8d404Vtx^4)Ms)TlH|hJ$jESm2P*rTFMd1n z-O&fZygI#a#ny+B2n)d{PLsgWa$rOv$i}=wg$CD`5QBNmLMe1eqZ9p2sgaP$bvqM7 zF{X07kh2MDP(BQTcFpUnPT%#}2>_f-?|WY!K#0|0RgxzOiNRvK*_Cq3S~_ijG!#hU zx`m;<5Th+P!8VU8D3wy#0kJqyqGmJ!6*oP|X!diIA$FxE7(@`@_clCs;P4@2!8*Nf z=<{%OGGO%g)}-bI;yvqk`pC(tT`qrk7la_gO1^vBNK^|Noh6!k2sS(Trnwo2fNcDJRUW%&HX@`dzDxW4~B_x|A zwn5Y{C-p~T;(~mnzaqeihrw;Y!kd5i`-tf<0DvwF69kMwwEvS>7mT#6s4E#Nc9a!{bk_+x*#ILWJZ< zdljPJ0XUy}5P&pAVJ%8DDIy&Y)q`UV6})I=sFUVS2zVThO4iks8|kP4Y$o&7xEreC zhYWfSI4=i|f9JEeZvJA;vN=7Y{dQvr@!2x`<*6T~0BaLcWnOPIN>-axj&Pue@An22 zjBcVqu_ncyXwdEg3)asUKXmGg2iNitrWg$tqB9M=_P_-oW;!lLGuOhyv2KKi)$s-c zVwHN7*w&eYxqOEQzCGvg{a16Q6Lvb*V!gv;!mMM*-(hfz4Co!cd zQkt*qi|POhw`ShH`%iti=@P_N4k;Mue$o1R?BJ5lR=qS|Lf_!AKdZ7D58Y-x7>sX!hr+l zE&>P)KdM(!!}&r;J0|Z>CUs(I!|d{Ma_J%koGRQmYE#Y3mGHJn&{Was9cN%rYY%wv zBj6UgmN5>>#2A*<+0Wx;3OH(LqLbBiLUDmY2u>?WKvVlx-2KtY?|uW-XL{c`SFD!q zoR$Xcl;Qu*nNEpD4+ls=Rx{kKwZKK7I zt_z85b-baHad$DC2A$T0YDy&+@>1)imb0Uz>*PfbPa2y5W9*-^mmYonsZAgDTxB~d zi0ID&|7!yh%EKYNA~;5CO7ZatzZ%UpOLE+-qlCBB{LVTx!hlmTZrWMZa5!Wk$>XfY z(h-QOZybE$gXi}BGo~u)WAp#x^I>s%xev& z>sg&v!RxQ+>EUX;<)e~g!ycNjDp#BRI@mK>iF!xK38|eLFOLH>_|LCy`DM%5XZ{2d z&+Pk~`mhxBQbmP0bqBtT6#;`A#c3K#!oE5ITXedI@_`(KY&5GfEPheovTI1vpv{8q zg7eipJ17p>slH;6Ovam4U`knY@8-{s&6|r@YI@&i)ZU;#fVbWUR&jFWfHF|G)^L%` z_$Y_s6NgNuP{bE2i@Q8JJ;@7V`#tGAT1EE}>jPbHtyw1Y3o~w3Ofsr$2;fFKbOV(pS9UmlV943!GA zW9rbN8i4mPjJ7eX#~mtmKz=*CZr;Yd>pnW%qiD98!0hPnh3F;l=-P3xR*}I93eK-* z=wqy!Sytg2httC)0r8fcJ$y$C;l?;rC%vc3kq#b9P32a#=4H||&-hX-jx|5)x z2d6{V2oWAHge@9?2eM4VWV|&_XbTCNHpw|b5-H8JG2K9%kmq9CcYu-kg!k({#2ll2Up)r}F8?xdi=p7SGQCFM! za)rbbX}i{U8w65R`VP~A~2fDb`y1_K3^Y>+Ic*_N<^n92^mUF zk`DW@Dz>Q!{#4wy>+=myetrmCMo;hi>iA_y(2c`u$JQaM5G}e*ce=I1Sf8OG@`*Iv zAYM-nWHUA<*Mm+4-3?htN%Gkw6@czM^6poQw|srtiO{)t#*GLaEiB|70hm|4hg{-VEW@0j;!5oV&nf4O2jdK~3au^CjNEeA zFUr{iZXvza2|x^=%H%*){m$wmPkem3htb?OpND))34gb*41oIV1cxn>mi&tF62r!9Nx4df$E`0vDOa4Btd+LQwGaU;g{9?aPqm^7Otr_rHjccEf({%K#=! z5jncgBom6p&tskts*;4nWgH;tw;mLih_+zuRi_2!gn8E zjHI>ceVhN(AgZpz8*lFi7if@4pQS3z5h2%<7-mW`9ciuQXhxNGie43E$OB@m5Sx$I z&=L*3VrW^7k&*CtX#%fG;`zvl-KPtleBrh4e(iy@->mBWA~Xvw>;jlGI%ibYgqn?XEz&H9c;&z3f7^x9hI>6U2XE*Ko`^@bt zTk@K_LG!Dd`iPf{WvDS2cEXg`MIt#dS*IC_O(?++azZe={dxD6*B?GP--Jl`*}E1* zLI}QYC+G@ph=;W)mn3e}NWv*{ja3;e*xFW3*H>V0<*{+F$NF>gGrzxl&k8UC%;>w} z^9cxz?A)C24Wk5c=qk}bC=)DYbs|@^zog7ODYq?A}jv5pChi!7co;fNv=KKaCO*lFz3=8qZh&dz422%{L z4QZC%K6J;sJD2pjtRtKLgIN43_zdolG&DTkt=9E|3JH?otE5&5i*`z~c|Ap$afn%D zHU&)_(1C*b4vG6!AdH1LR|_Qm&F5+>d$BA%-#OVmYqFqSJs7) zPgvju&sLz2cC@SOGJMz?LFaBIm|-2mAmk+q6pbt%?6?9fky`@|Gx)&@u~p$Knfmge zPf2PHv)}^Or)dvJH9V3w(1-j00azp&*l@T-ThUGAre z)1|hUAmeby?NkrH(eE>2a0L=Jr7+0GhE&1M#3+f*l#}TlGMSzri_HcyW3<$fu#yB< zA(raoF-{tVmMDTFu=R5{{<>w?;U3%<`J=alp9wqv0-C?7Tfz>}a)V?Ja}dL*4TBC> z%cO~DAYaY@V8`y?=Da+0@BhnImK62R(caav3*LV4RscCvsF;e2UB$`A>*GaFZiLbr zQp7C;ZFMXZDF`|SbrBjMQk?8M1DA)*%w=S7T`O+-h4d;D}q`9KgkmipCobiz~-rpv(VJKZGW@O-w=kUH; zp8RmfGsk<6zh?E^(F||9D+qvXW(O|h+|{^eETqZVGZ8b$V8>7FeC*pN-L#gu1#=W0 z8pYDEl`)GuH>Pu_6>>k$3-Z{8vuD0J^mhtzhIy49Fh_)^t&&PZxWbTun=fd&g`#e_ z(ogF|M_tyMDjH|S2Ri9WrBIhJ>kgofvv2;q>+iE*`#P)7Rue%e-KU^2LDnut_;W9~ zOuEWJL61ny#EOia78pz+EHQ1+wcD-{;PAVjeeR>PFM<8stiDUCpsN?m$b$J}=d=sK z^C0X$0J=tsToR;vn1ZMwjJEs8{ZUQH3zwsexW{I#8inIor%2aM$_Rxlej)~lZ9efrGi6tIMkaDTIwJ0{;HkDZsO_+tLlOsTSZH-iDwXJE+fU%9M@VcM@v~?+= zu6|Gqs^X?OpRYT*^&kKhX7nw3c80Wf+8Bqgh|UN9m{;3`bOoQ*g#t959u_AWG z2fm=q`iC{@03}B%hvi+q*yhRw$l3~y%4vI}qW{OzS1{aJTwC|{HlPnHLn8Pknazn`d72_{y|kbl3?nm)GpuxSxuVhfX|Nm37t~9spGxj#6tDfO?{3 zSu-}~i8Xv~Nzh`m2V?QP8f*lqQO82LRJUlrBD&zkEl+*&#s_;}0~B!c8s#DoLwxrBik#m>uG)OH zDl1KhDd~#TW*i`r{+yJ~dm$dM{0x@4qDl7?Hw>g=i z)Zp#+oc+fA$8PLNx+$yl+V~)j~nF#z( zl$LpR|Vk#DH)a6MIy@s#JxG_{c`_Rif{{6^-4_1ZF&-$yBrH5GG z1i}hksZk-Juyi1s2&`7H<%qIP5?zI~%wi(aSdLllFSSw$Ta-0QA@Dwa;Ke()U-Qfd zt95&!!;h|8CUX3b;bLztEmO%+2r>Gl>S+i#F2TAjU?piB)^V;&83{NVc3V0j3V7mZ z12YyRbH;_7sIQyS2v4x=0y;)yMa|3tf= z;}=So4qAd;^Ou0=))J^3O5UUr6$evD)j@ZMEXw2(IGWr=Uq?zVXi!xB=%%w?`}S&- zfwpOF^BXH!7#q2JS5uuew7f)PIfk{=9i427z%8mMEw_zL7r?5Cgo8vHg_k7ghJ8cU z=FT&CS)5Z;bGN##w6U$xku5E;sT74x_b>a-eeRss4qUhT2Q2odXgG0@kDZ_Z)9~!I zWy(b9`eZaQ~J4;_e=%+^d z1R)CzthYZq`@}o1Hli{6_20ng$8z6Z<=!9Jp$!Z2FG&*)rGT|btyE1xX);kK;*%qH zJD!fx_2pcqjV+PFw-mZ7NEt$ISz)){&ojS~XWIJ#|*73r1nU z=HfAocyJ`KNi^P24`TL)y;~nT_r4A4b^Op(Z6ko>PR7Don@}VRT{Z5)WkT^Wc!T7O z8V%JL4rzr@(ux-K%6Y9ZS8ext3+Yn2#=)!Q2p|Qci;cgo!_je zTCd;z19V2%h~$+8++YW@=Lz}pY+jAvX{*Ca;7?xp z=KW9nX6Hhk$i+(AY-EtM zlqMKhqEPhMf8?JZ-gVXX4FbpalPgv0KR-u@UW0yzT;R_`M1laj?^m06)`-<*9VKG? zxrZ+FRcbkYK;|bW1Z->@4_V^{_kDNU7iWEHL1X*SH%h>}zEO=%1r~Ysr#LhtX1P7| zBBpBAdB{t;2G3Df7>I5L@Z^oRU4Hu2k36!$A-y^J+$upr*F+XMXAeQ0{a;e48}Y{# zd~#h{PD*oRTp?RChUSb#oGazzl4eXrQ4$zKx#CPR>-4i3uh^&^EX1yAAvUt&aC%m! z)Spkgan^!pQDLN%k-IonD!n*wedeqnvfQNCz%U#6prOngj+&f`-M1XesKmA=uVXI_ zK-Jp${lf=#fAzcu9qIaVrTe8Lu9M)BNU-vx6IzuU2+*R`Y=VmwCpFe~bDp%=0-@zF z%L{9Ykfm?G^v@UHb=}#hwsh0leS*~{+D5h?g#lAAYQR%0#FD$XDT+wc&vkYB` zEEv0%UA2m;i1U3@GB>1=FVoo`uRYcTLhuujBoq+0m9{}N=Q0K;no5$UYw7!XO))|A zR*I5DXD%~%$9WtOSr^8dqK&DJ3W{+@2c@cAH($2rzUyy?d}Y(xk3ab#I=_EFKDiqz z<$oO0JHRgr)n}4|LA{hW43mjk;8`}j>Jc5*&E8jE`TqWIP-_vO;61Ri1R)`3-}HRyYUYbopexMiiaApR6?2Xy`)v2-PE$ubop%7#oF>g>W@6~E z3?YR$;-V)GoVfJ}P=jw?yYK<0dj0E7D_~TKh@X27c$E?(QG??T3fUQ5VKHzPx`Cft zYIc{pL@Q|#MrCDR&fXX`V0zDd=;(WIdk8eBo7ZrEgx&aggpH2PPY}TqV%Q(OQ8J<6 z><#dWq{T;E81*8hL)Y>R(s_=RT(^+Q8D}Y28%?!Yk-aM$mnIgyS08DlAz|M4(UFh7 z@y*r0MaM0T+<|`P50MYkSEH^{ zB_pX4C!yUiy9aV1DlD|6*9fUi*kK#*z?KB+izIS?00AR(HZ#663hwyyWBY zOzjmydEEQwe)Ys_|5&wsPHY7j3J`(uLx{I<+hjL|_^M1mg(1}OS{&b3OS5%yQ+)_! zsn>3N?!z}OxdDwgF!?wG>Qcv^_9JvktO%m`0n|MlE;rIahz*gPFW|AF4MH!QsIWI38nblvhA+RZaa0uqygM0&^CXHoZmV}k)`_XK>gmsp*`TrPxMGiLJ@`}*O<1+aIX^xZG85g@W6@6>dV&g%gcU|IB2MWPxtcl0 z)JzDkJ^kLBFTDl~<~FVUzh9q={#JrqdVE5i%|pWFvcT3hN*3lOOQgs&urNs$ zB@YL1kIJ@%m{uISRDv>}uUlSlrrtgu%Ntqw{02wTVwcEeKC3k|S{Fv9OT&<>i^Mo~ z(PxQ~tf;98yG)*;jTV3_=r@-?x#iWzk7J-6@ce8AYX1V6?0N^{QvgCw3Tf!F@C2+f zFE;Zq3PFl#P@F?Cq5`DazAxqF@uTcvhb8A~Sz>oY$rlH8jl9YZIoAmamEKJwkcRUp=X*g(%msGyz0gjR-(AidHuObDPK;!KE1*&qc76c>gye=BhrxvKH)CX_vlEwki-eV6wu2fX(e^pWQR3@WSs`qv@iNa8_Yv?SAI@pOV4XrwF*T0K<;ZZTmoA2B+&rHaD7Ca| zQW2>~eWWmelP7Gu^QpgYJ$UMBEoDg|(4D`dL6>{P|n5D6>&^sH_0`U z5j)3cd!_s-4LY$Gh?P=$;16(_o)}0`Pn&#udbxK zPf4T8;AhDBr|k!gs=qCxh3Hegk5Mtma@f9@7|=vaV>pPOIPRM7PJH$Yl*M}JT3WhW zvHpiYytYv{EF)L`H{ev9yr>qp#=2NOUEg6^cvutB)THE8#U^7&76dY+fXM8RBxOdN zux*_eq@M8wA6$LLYX?8SexvsG!ueVC`pcjE5$z8f`S|V6U>{!!FhfF3k*n5|7Nv5C zNm@!(Y`xR%Rj6GfNi|W1gE&*%-&tft79J*L(R#FwPQ<~3isbv>Z`t+wcSnQpanst@ zntz}HghPgBaiFwoQRba_dcM?;;(CL+DrimjXlYMB*CFTV`jjLpm~EfmcEi@|(6jK+wfd_#_4;|7CpX4{gBWj0qqHGQEH9oCT>J$a zgJ0-n^x?c%tl6mS1TegS;=KRtC$2@|L#W~%dNGuGU{JjEEqkWOYQyG`Br^=_e;1ym8yA!PZSTp&|Y!Op-(*zO$C^P2Q+G7u(E+g{VJB*CgcT?bbB4Op+0&-@8l79KiPHkH)G$N= z4EezIul)P+GybJOH|P^s&neeWe*Sy3<8V#;Ep!||EHTAg3s=WeFQ&3_CT}X$6&wS_ z-*0N-c8z|akOH>(@(r*5<<%X>LuGOETJQ)5Aibx2gm#vU9P$Qe57;cpNEpv05=K5r zn~@H^$%sx%4h}j~41(o|ymDKQIdpVJTwYcrk>cQ{bM%MzUU1vTXKdWTUKFgDitC8Z z1NKp8%+;_h#n+RtdD($y!6PDHY3TYm#hQHT2X19)S`YH~EsqWJO+;6$FZCX*M^`?E`rOMF7rz z@8)-(eF3&Xa2=bj@Od8c&JX_=a$&@3b4nz9cfL5D4M~{!%#Kr(V2il%rI(M|`4WJ* zhpyfDEx2&}4<*qD4e<8L=o`ELr%b z+l}-0L4%>N1ANXSo4r>U|6Q#@;p=y^+lJF}#B>l}P4 z9MDGCiS&$OU?;1kF{5iGv<$HsA+~JbEx0^;H~{?cwzr=@{Gr>)XqOmAtJUj&J$+R^ zDj}vvfzERnv6PTCn!((dxLoQPJlQ;=Sg@N3KPkB^^=wd6*nZ!EFK;?)C!`ZV)%q%q z#@H0O@da=X5|c}6mVhAdEi5<{dd$RzF`fGJ9TIHfIU=^;07WuFc!Ozx9(_zkG1x2#MVX9<*CNTY2EIkRN=$6E3U|Z=LtKgw^2Hp@POGSAnQ#*mUHHd#eu|0+#adU|-Ukp3$7%bW zz`c3;#+&as{gN9t)UH1`^+`196-Z$R)ClY`@0>M4NK}5+Kc0|1_VUc4v}LW9daGxs#7E`43BI4@vmRRm6 z-yZSo`GFX2+y@`L@6Qk41xE^4Ev8|`KI&YcZT1mM0noJHX$=xSKkOD&C>__5hmnNK ziS9J6YXE%o#TO?&ck_cEp#=Ry*M55VeKx^WWgVAE7GLUT9A+m^FlXDtVLr%gCjjTa_2T`{?mOpR)ZF3F zwY`7Ysamf{0vpwqJaR?ma(R{@WjRbm$tW?M%tF1Ig6d1UrbN6_$iTK`VSn0Z=b(vXsIQ_e)b`KqoE=4$vGgKV^1rQ(5&4O*Eo~bMA#Nf3nnC&sNjtG zQKZvb5J2+ej|T}DYudA1Y6Q@^Jv$!0{f76SgfO2qfE;I7C&&Ym3Vnji3xJW zW3h3_9n@JYWh|RE@$rlbp5T@hI&K0M9#!y>3^fu@7HLp?Bv-(Ymc%JOt`P1~&pdM4 z5xf4r^Au=&(;7*jL5I#Xji#Nei&G57 zXrwkd-F$-;Bhc}Aqg4ti#j^+B{>!dM?^<;MY2S}7CmLc6$$(H9G50EM3$AJn;aj0t zr2+oHDLh+h7@3SyAFvLNdF=at-TU!3=&WsC`|#ucQm;RaS$Q*7kT+GZPB;r8QEkiO z^LVV>L5;^V0cDD5VY)gtJV~NcL8+=y4f~?;ImRZ`$H#I=FSnj>)V4EUzZQ^EaEy@L zhxR6dJoM5Lu+?MI`gVcWGsUagyq}Smw+QS>Cd*tvs{X;#kL-Tyyubef{ng<=Yofo> zAn#KEp=TAfi@1PEvQFCSyig;gdYUc)#?=v)gG#?~9G&TtJ&TT_UL@NCUK66`n6oZ? zw5+30Q2tsksewhBdXU5sw|&8ijO1042x0kPwkySpTU~fg@sV6 zrC0fTL%(y(lp?JzlULJPwb|S_%_qTefB#e4&pGY12UfM~-wc0_MwLdG6E;zPfeC;Xrl;)4V5Fc)tW0?exG<1`IWRhc(9axq}8B*NR%P zb&Ab`X%&SsrfRxWc^5fLDlAZ^n3a+Y?s8k+diU;6FMAGkso%VI=WjnztzR2o5tDy| zgnvMSg+l9Sb#(pA9`Dw&V_sO!RGKCourXI6k|0S`k1>^fJ3AI30)T$>6~}*j*Zz$> z;I+nQ%Js+p^jmZkWXO&ypm3+Owu4zi2Moz-3d5M-pqq>a5+@i_&>7ymR#@O;{Jnan zlQASbeNUJq3ev~QJl~k>3MC;4gx!`8-ro1s+jmLPuhw6&@~8PBa_5dEfs?ZM zb4yDqP0bVg)2MlbS;kt}kVS4K@`^nh%+EWgeE;s=lfHa@HTBQYBkJ|t&PO&%gBk=U zGeBt!E12PVbiYTZZ)+$-YB|87V#HFQs6O}H!Sims>Cr!-QB!;tY#5FmJ2r{~>&OKX zka)QXLDrD9s3NtdtydwZwWg}kFp`TkZf{Xqpc@@Zrm~T-j7ds*NsZ7QJW0=sm9%Un zRl}SRdxvSuJeVkhrlYMU*Zso`V7UK${IM&K|DJ<>?v|~5G*K`j(=&gj$YMJYy}+3X zxO-T6v&JV6(t(!MCJCLLF)*5@8G+76ifLjrrWCtQRx$%?{n zEX=Vs3QXg^Pe1zX)t#sl;pVlEcipU7e~hv!k8_c2-$Hp@TEw|xGV!uQ&Z_u9VVE_N zh4lPDJDPJdQc|U_-fVR`MZLn78VcJauc_9VkV!n8nG0Fl$1k3J^(oIky8$I#b4U@5 zz8rGVWuJm%OBE@zy@rCWU_y)Ma`n>BqiX7Ekv&>7r&Dc*SEW(e(|%?OSU?{f`PKt( zUA=3yS5+@P1Z_r-{QR)HAuT4U7;UaLm#KOsz8o0KWr#tNy{?`2luLqT2z3gb-zjfG zxSjjA?N=Xo^1c->$1>)S1kj&Zht~M#@HXTiko;Ke7 z=kHH{_?$1U`A;;|etrk&uQ;Fna^sXGAul`v1>E0By~C+S=OS_AO;JJSX0&5WS_s_s zP$9=D58UzPahH40A0Hw(T(vI#`#&~999)3^2^Vhukd0Z|#aQ2x!_Dv#kr*Y~$Iga4 zLfEV9#r}q38NpRMLxO= zSb?rYik`Ax)CjVgs0KkPxOxuol4_GDa?l{W<8I(0PW4$O5UZD=QWVop1lJJGb^$f(i4 zq|Hv)3%7oE(TDGSd?pH8Y+n1~q`#@wA5SIGzWoY0j|*ost6M5)ED?;YX<}kaq7dk1 zZ5}mNM>a74tm9bU4ZWp?M42wNNk&2G zrg*7+s1Cey+LrxKU3&NiqH$jRwT<)k+*?84%nT3eI0=$qEJpN%Pi#$c*&Q`~;q4~e zONlGXO539*tAv=<Q z=*NaqQlF4jtejCBW2DBOww9);VOEq2x`|3VtJzHjf+G6(xp=K`Pc-TbYlkhz^(7p5usm>BT%5 z^_cTM9{~D+B-`Si%m948^59=j`PakGt+xJ8dk>-m^m8P2Z5GrJ>PRflNiTg?H53fNW=!6}OY^;6@j@&P zrqk7uXVxiD=3=QWy&QHBf*Y4^(6w-L-+s(xXWVnotsB7m>l9#$9Dmp!g0ax&iy74XqJP;*8lxd1$w5$;Y!PR{%n0Y^NYz!f+Id*>ZhOWlZSJc}SNhs+H#BYy|^Bwg0|d zXFj^){=Y#m9J=<=C+q5U@`npG1(HS$G*m&(|EnMBmU>}AQBdlz;=x$Rkr+t>X}et} zF9h`{Uo7QTEP`}oKHh!88K+!(;M!w15Yfx8gnayJ-%46PKyKUw`ba`CkE;Y)OOl^j zNG_~$k=aeB(TXIx8{Dg04l*#is#$$~;^#XC- zH^Lb!sY!-vU5Gjs7cWC3@?#V2ASkM5%sd+lZoi~HhQCB7`QlG@{QazLDEn#C+9j8- zjCOp;;g3O$-c@9gr7JEkLIO-amZg+5RBgl?2?ixRva$jXRX}a?w9gKnu=DcAS8(3e zn@(1(AH8PC~*if>d!aaJOV4i$o)dE6e7LuAOL8M5R=BKu+iYzE%Y} zQaPYzdoMn4?&IfPw4zv)e*Z1H#x2NU#Gfj&6of`jQcWcZ(GC?ymPnWw^CHCNvZO3V zSf-kuDJ6yzVL=_T_ggR#0 z(T;>HWVeISgqiNm&TQ0}nG&RHI?cA6Qua!^wvJp(Zww1{yTt;x=a>Ke@^c67eEX)= zxIOe+3%ZknS>zdJMOLE6y6XT~ruiv{A^tGjl2XiCl+aOiGUi}2>L$jP z(IU4gvT3^dh9EGpR{Fih(A$m8D9H3*YK(v0d-)J$cU?OLTSqDGSeG z4ZmFFx=ZBaOCc3dc4bq^*y~gaGX$Z^;$}9h*7&%P))?9gS&VKq&1OE#Os8^--A=R6 zo1~e#?6^a(V@9G~GF+11f9tZXFJFBZ5CJ!>o%PhJM!beRs{u?)NWl9rJgzE`=_ow@yiu#|Noxys@89bpT1Ed`xElL z8niMhd`CA-f>f`7S4L`qVK^@po2RY7NS~=T+;qK<*&p}Va%`A3v`lS5@bxV@andG( zr?C3wxgKn`_rLJwB^U17v9bH?`vbcB1d+d-4tbo=K8)kdR&i2EFh?B?9;sAJXe7!~ zA(Sr>y25lrmZpX&I1PNchg`|b^A&`F%DH!*c)?AVyt@S@MSR~v zzCD}f*U0*)x|)m53&wB|sikNAB2Frxa$8gO)XEPc`e4JFW$U}?>lMqFY-4<(3$!H^4AXo>a1TkuXzJlY9LFof%IZ7TW0cc zk21 z^VPp7o?21<8_3gN;Z<30$Y;jL(kU885SYrO1*j4w^uXtM-RokzjdgM$3Pwnpd)DuA$BQ%Y7!LyimAMuZs95{3Q|Dt19 zKzPPOlvx6=OTlo<28OLl1sgX5f;Z@`3SvUUYYaRwxhYaq3=-old+ylz=Jf|IJ{!7w z$XZ(klz3G8BShX=0;tT_n~OrDh!PL${asywDQY)#ot~Gi_tew3L^6Ayxnij5x z$2i5Fs)a^ZlfxLJg=Fx``Iz{kZ*A2*Whu!k&v}V@OlQKelLN#))@v)$8i7XDj14zZ z^O3eA4Js5}UDlA&;QfApBlg~V!<|>1cJ6zt@z0&OhW>pSIcC!~xb+dSEb-hiq7te+ z?p#6>$lc+561Y(;0hqab_v*Vl-~Iksuz&@o#!ph{?`((~knk+Ls+xw;z=oTKvTDe6 z>MB+>Q!}H+=W`3SoJou9__0QAta_vX-R^qf>?>b<_}SN2|KI;+MGEgAd&mGJ(HQoDETV9>J0moMCK@FBFczG)5n#j0Fm zL(VdGV2Afw#5zhmq3erfzDk^x4=*SSRnnt(4(-iWR}u@fI>L}!6Qb+Xk?4>~%+Jk< zKxEh%b^9Dn-G_r@=!>iG+~@|!Py$cg^+enJNwry+9OkG#+?d9|Tl;tNyXlBzOQGO3V0 zip7>=HoKD+pyQOyc#x(h(BgQ(+@Xvr%JDe6AkvI@5Kvw3h}#0Iysc+^``53pJ&59t zo7Vi_9ES#U7*Smw0xf?^7q=Hfb-Y~JV&`4sshsS}>BH=lh(^ORW^qkyG?Vyct}=}! z?D%svi=-!rR_6{fZz@RG%L7%$1$VnI&VKoud!BzCMI1J-J^B$ioBz)zH1r$zh!+79 zGkqi<5o(ompE6UG6tRJXy+@1?J4&s1z$UOUgRp~-CFw=^KCH;{0tu%kLk+lqO$#RR z+{Ps7lr-KMO zC)nCm>hnXS7&}v$MUpa8j2BEK1m%UY*0E{j`iX(1 zf3gn^Wis;4Iky9;+1+mp%dMshS9CBq>QQ&b9(T2s7NVC@T&Xlx5EwE9a2-N|tiiVD zT6lP+Ki+cbyC;2k+f$!I4P^7$5!>JyhU&>HGyX%7H$>o8_lAP_IE?pcQ*@9g~uWIA@aHbqq{D)QUCYpxT4GC*{gd*ySNzPDT>zi67 zePz*#XVOH{XW)yG@j-Fn=GPaO5+F{?dcBS#5z4h>wwO1Jc~c%t zz@b1ma24CL`;mKZ_zPglo7VXM^`Ge8*FeYvD;)Q9R&0?~H7MA$pxY{|JX{x*>X?Lz z9ZzPM=LM4{_%u7$-=q3GOF1K#TQDKacV2h+vpbJFenYSMf%aK+xKhY(ixNJ%2KL~Yzcy1PZ{&rCXrdAQq1N_o(|E;>k|egVU&}cMfnC2 z*4)B+v>auT`#@(&}A<^`{-&8&Ty}YKtbfMN)|jD ztPr}ixh9j|n6U&3hl)~{lM)2h$VIp5D~zs6ppKhp1{>ccZ`-H9%6;t2vu}IrjwgSO zcJj1etk}&Zr1aY%$RYhfl@gy|DV9R1O}CsN<6fF5$m+;pEiW^tPTPS-M2Q(F7LtZh zFu{*H04D4{{M3s-KYlypPlv7@dB{6ZWxVIqjSE8x`OULX19r4{qC(ncR}W<7e!!GS z;KG@@Ny%jTt!hpRNsjBp60OXM`D;$s#LLv#dnrp+IMwWOz?AU3Z(`0Eh7H~YYfcgRDYK76|}tLw>^!fCdX6M}?FZ}Bw5Dm<}U z*VRG=K%a6gSCu1Lrm@MYsA{nZx{-p58tjKOe9{OWnT-!*9w8+A*Z%vor;oh-va?pB zoxR15M#*oG(yumw0+lIDN*WaXFq~s(?P}(9WSfC!p5Dq*yN5zT%H*HTG(we^p20++ zytwV-XLf8o{IWyQgmuelTQ@2;6Xc5j1@$CTCN7CEwq!?0bw%tMKfX|R!5b6Naoa`e zSv1|g-#>+v+UHd`mjW=Iz2K!IUq1ZBtMur^W!AvJ1%2PM{oWDEEN7r~PQ$?f(lmGt zRZCA4A&!bgdQdtisSAd_T2gHImd43+VPa9M7={<%ijOY-6zBfr7Ya+O+v=xxSDSN^0eP^6>#DN=EKmP60E0nX2 z+;RXsE0t_lna`bDi{U;yUvvjpLPk}m7Z-|{F=I9rv>kkv)4|9j(WWwhkc<|i2uDWA2zo|>D>h%xLy&e73O~^-oaFG z6Nt=P!CI~u$kAP8rKIK^<1WH#=Xw__@ozS}_+le+%wDv?DG9J++cUL`j?WZ511+YzPhSti7 zFp0di_BK2UrxyrXc)m5|M8ry=!ndeaBA%gc8A?aGJns0)?=E@#^-r$hqif`bhp5W+ z_{(RZT^b`hHzz?~6m-&?B^+y{NU{ZZS#UC=OGJE(0D)ZazJDEc#98P53XNCuf4sR7 zuZJNIF5&ri6*)H1X^X_}6dTelNE(wP5RQwjBSJ0}lsHn#%+Mwv2s#8##kzPp$I{TXY;l8V zVHwK>A+1WV)aTL#BfK6$2@}ZJyoVs{`RZc7nWd1;*ecniYXn$f@7BM6_1IqyZlFLU z^QwrgM$R`u$Y9MObJ<_x4NUwhN9(AXicN*rLT^d5jRYsnBv2vKzV*4Azj*kDo9%HM>rY3A;rGaq zLbx{ImSeAEnj(anks8l0h_wB!;|Jb0<$UA_dokmlsq%qAAYPf$?n^a`aH%%KeEEAD^s;r$nFXu7o^UsGnE z-+0T%i5zZZ!RsxGHjN`8a{P|0UKh?;>MA2h1%#bRY&J&JYP}t{WF<~t6_2C{fM#|+gF{N9CbKy= ze_~f~oa1JakTHeoVWY0gei9{l#Q=kO;^C@me~S*16;a<21g1h~;Te1IJjS5Qba1j=ud&^acbFwV z)hB@5HP++$CVpo^nyW}MSdy?wVG|@E?z;D$r(Zhlgq0{jY5$*%?JR#YXf*_R(HxAe zL2@jS%8{~~P!4lDQG3DJ%`wJ$qoW|=3Y6};yr1%0%C!kc-NLwokfC4p`i>9pJZ^^0 z;+cZ$(GDh&Z)|Yu<+$B5X@;ZeR{aErALpOResCe_ zXRDBgU%U0e=ePg)HaG+}t-bTcPtXa*BKdtV!R&rooFAVH)sDc7Ol%`R{!FD$1jQM*u!s;T^w2{O|t~PS>Gn8R9l8 z0VC8_tE8APm5_C32Gz2ni1)e^JgbpgPLse6oa8EXy*wO*^+C* zkk)Qq+ZUv&*MpVS%LKnc-uk@~O2Y)KsvS3jD}*XtBN}z1px(tVEOTnvA~=uK24;P< zFOq?qHbWmlti3Fy7*NPXa|}Vb3?S|ITY*B zj4X{=FI2j7>MmxZlj_Bu1F!RnoEx_PkJ%Q0LeLnEF!mZetCh#k>I{Z4Qr4da%4ySu(fj`WOM= z{%5{<#HDwibq2%^Ja2l^d8+lVt{94ftxi?!M#r$G>`Wqa60(X;-7uQ$Q|iybiqJF3!)= z;R+2LEh(n=xe|DCsjODb#W);SUYl~LW@6JkuRHNrhdY1?qBM?=Kl$@LN1eGqKUhcq z1sZ{MWb=!Eh5HvVgr@}v3WmCE!;vS!CXrN@2RMadw_y=ED*XH;M)&F`Qhhqj;)Sf{ zsI9;o)`HbuTOlsQh)FobuYL6H8$NjV8-O4-t-ULipp9=vGS7p(sk8|^+?p}k#rfD4 zW3yTaD@OXJC{51#OEGLD(9zDSW*M1gnvOb3cqz0d9nCehTs#7IuPcxE=Ex7Xy>`uN zOrJj#igHrxswD44o+LwgQ_!}|l_V=(Z})J$mZ4rZ=?i8WN?brh)I7p4)hW+N7EQC% zFZCUEmQl>GYJXc@wznrZ zIzz>`#o45qJ~k&hnGLd184ril)nRZHYq}d<>0DvRJ9!KwV<)(}GE&x)Vfs{nZ`^vz z(NFKa{dRZ*VbhuvvjX@H$Zu3j$BbG*5@sYD&i~u)jY=}>LK+$w1BA+d5i}3= zV%IvW`8(9vY*;|%(S@2k9_AxXZziU;AdadtDU3O2j(pYx8^9B%e);XmJ03!51Dn^% zTPW)FPvGG>Xf27n8wRTffsGlWP?jDF(=e2B1Xi1iYayAnjBZye^au-0c!{qsCmS*@ zCc{bQ@dC~nK@X?nF(2-_`@xU**wK&w$62e5w1mtLfihMTlNA)pn65c!QdQL~J=HZC z%X)-Vs$TMg=?^dyYjQnjnr8s0U+b2cjkIim6n*?D&OgRFGDLxsAnt#MdM6HcubEz;!p=<`CKxML8r(UR1Uhv5iXMB9h(HlecE84Hop-LjBe-3D< z##H1j+I$at;zJZ(H{Dtnh32khrBN8S!am-J-v?J0Bi2S$8d4TKemHKD!`x8WXqxLK zBi$%bgz@{}rOVI%;1v`e*u3_i(>GP?%~PI3|AL2@_L~92;}NP=N0O@6Rx{XP*%mFv z>4m`*TlN(VrBNds^AzKF$k1Q@;LTmHd~n|C;&A4VUIcRKCoiG92N4OJ4q976EF2uz z**s>qp-3rUk&6pbt=Y#TyE#l#L*vC*ni|Nmb{&1;#Si}N7_>|S@7=z&vfCvfszDi8 z?ZN7_qH0?W*a}nMnk2~vmr;g;w?g4Wa16q*KYVuTj%Tkve?vWb#jjS>6bW+LuY}61 zq?QO<%E>~>fafzgF33~!Vhq1xw>WXpxzjrJjcY8ddzvoqOR)GBTmS2FvdSIb-7DEJ!k2iS~8(!qeS@>!* z6$5YhLc;j*p8HQe>z19UkJ+JXKmPFt09qV=58SQx{zw8H(IfC?G$abHqQZlBH57Wb zE8{m$nIa{>;Z%XrXkH+M#mQ38>v!|{#=MtSXsPEAQ>VT5#nBfXzUS`MSN*(sIJ#gc zi1e&q0&%w^%G)?`3Di)_dAg@a$MtMu{Lm@aM`T$|NQ9HnYMwSp-Ip4y5wCx- zoY3GR{L1@BANcoMsF-2%+RGQ1RO_c6x0*jW@>3E3@cL!DWLeg{23Es{8--)KkcGB1 zQM95yC2X?kltXC**KxL|c|xRHA$f@=RcTo9I2=Q_$3NiHCN+;AZpG*R^U_P+KK2&0 z*t>ZxaQQF5K;Wh(b^TFKY(Q-nG7yY=i@SbIq*jJUVG8S_x}Y|i$NuD2M;~-4ny|5`ZU~|8cmwmMr4-& zV4zFOcv%Tx*&3-#!E1t?=wfDBcOL%83Lg+s=khKo@4l~m+Fg3)d81Rsjj$N<_dAvr2 z)MykDqnRSm#$+Xd!OYI^|36D_;W$@ybq&|j_D%b13r|~~mp1LVySux)-*I>MnKvb* zI0Og~q$GH7C>EqZAh^2~4-kTr0zv!jJNf>AIdksOeb!!kEvs9TaX{ki#aI4)@-wf# z^TlcoZ$1-PAs=7;6SO@6V(;Cn%(8-REXCfbDaylmXebOhfNh2$dX@%hfuR-VGlMi8 zxUI>uH`X1=8?eESz2onnU;f4o@P1jpwqpkf0e|$TJlfvRkk^Dz)6XxnOn9O@(JT~f zWV|pxs9EiTDYs?dg+VG;k+kzHqoR~jh{js@DyP~bQE`d!-Ittm+<~i}eh4n;^=n)? z)ZslA@u7_}kPWXN4z|K7T-#A2IW6QxFQ%9$a@kIV&LZ+nxr9TNisAGmEo68q{vT7zXx!+J!V_$9%RC6iA2U+BaE>Rd>ngs zSeqzh5)qE$)@8AVX=-Gog>v*NwMZ-vWroq*NCW{BKD_=mf3^9Uox9KgXWiPl)Rk%b z82RKzw3DPF4qN9DYYQf+$F5;pL7`h+s)dZoDMMAyVJ-0i0S~5X!H&~dg?ae?mOG!= zch`YcOTWOb=Cp$tKKl>|EpaTu;>L+mc0AiA6&F%^LLx2B%gx!?AvM-$eNZ)m zf~(jlw$7-^TEMC;%NU}7b%OW76S_?px>bxgm8=|^yBJ znRJ}ddJb4ZZ+yJ<%71@xDLg6HuU)zSCe?=DU$SzgP$0L*ft82^fzw*2NWGMzi?7ud zw835?&2Lyb{fW)#i*DN|lHzHQ@js^!TCom-4d}!_JGmd74p~^$jZ;+#-StIXF|dHzM`$h~cuZ zt(37Oy=1r?Z%#Q(reZL(O)Neae;9~NeQsU3+a%ga-F#lR%m(NIt&J{_Ga1dAszDa# z;FR3;&6_vA_{=GBbb&{%6QLmu4!Ld*9q&N$7{X<*28O7g8$2VHMCEDqyEqJ%7ye zPyX{?Xoc(gwK3hH-thjX0kp*>^5_t#C^|nBfDCLzC6yktifqaG}75cFb3Y z5;K4OV70`#$Oq~70G-CE2b~5VKSf+VVlf!ixTH#Our7ck0K@Gy`)wJyGfW6`@}VV8 zRTzQ!gJTa z#Fa8QWali(>!|b#;xLF1LNx;ti1kb*e>>HIwesEGtKWO&vX56J%lprwy6vS~-rn=jYKMkTxD{<#gZ!R$qAGiYPqc8^xhh3?UQOfa3PIOaBPq!-cpB_} z_mfXfyzx*S;PbnHc7KugA#?*AhS-k=iz=5f1w=G{g6Si*15vSu=faL%4(x&+w9f3Y zWm-pY4MrX#v*~1|>k+Yo6f(POY(!_sXqr?p*XuX9JTy%>L+#SAN;RZxKmGpR3y%I| zy8)f(Kfinl+Q}*MXt)S@hdv)1#*GZGkR2q16bXizR?y}bAR4pnt_#k*`s)+Gd;QmI z*F9`iZrHdHd#E((=g2A7{1o0u4L3uhP!T+G+M=oT`kKCcnpd(!Vw{14JY$u~j=YvM z5@D2-VI!^^#QEU5JJhwy#-W!3LgRi0L`wT^zi{iG*MQ}{ZtXm6<t8NFaMQmjE2}4`znk%oa+pqKat}6LXo8v}t*ft}Qg+IYmX7+KFValn@h= zP6l6OWr_Hph!LAsM)GhTR?2lJZTj%&54No)?(62S(TQ^+nV)+>dt}Or=88EtGvdq) z3|8-@ z!;$Out3hG@<9#vRrrssQEp(?g|MUulpA zp#41|(6tbcY&_}Nt;g)!v6|5@f4e#@e}v?9;043h>$w$7p%t;^CsbLI*Afq9OogIY zY6W~Ec~e~SjFmnHqQ}pjwy2E+c>Z({P&9SUV{{AfkAjUo>p2F0Cp zStm(KSjIM9L=kuJB3f-K7xjzS4RF3G_gSh6J>f)T;z2eyf|G3HiTn1P@&-DT>(>6e zTZGOj)SsLM`tN3mQ;C-}sANK>#dp&=;GOAnw{jB_t%4cEj8z1;B*;T_(i9i4POW%T z*srV65S7C36OEh=z8x9zxNZ`<{B~aT{%iNV_3)RgIo|TK5c(%pbyX$e2CahB%&LW8KM&V6~NG|p|6BSOfMU3|*R zH(&e>svrsSBbO7<*44;6x+f@##Chd1K2VieU3Yh%AnL?)PY`AnrJxUS!+NaO<3S+<71FeejLd zcS_DU9PN=2`R=Q?!A;E;tIo|%#<%FDO(rkQcrxecclxd?itzmOU zxIxpxYt-6ODTiJ#rWkw?y*Z;UgLQaX{p+T$uitm&DhYGnKR~1pYH87a%aK37nS>LS z+g*gBR(e!oRDi;|mlT#26b>35j)8$Brf!c7M2dpRs)+y%dvVJv$33+9r9%iup}*pO zz#Z~7*eExq@f4vb%P=@#KogIpeci<0`F^oXn)Ux~GQ{j?l2 zPUJFGd~37O;U|Cz?yJT0iX^RT44Y}flBuV!F+oo@lcLrPMBbF8wvfj8QPJ%3a^b=F z;&I1azT>e6A6$*j5x-j17tI|1^LADH-7rnjhCRh=IhsxC+|~i*kd_})<6Z2>nu<| zsf?4bLd4v43k^~Xsob$~r!E!`Y?TvqGst5NTTM;~Ft%XO6l^ETN{gB#Y>~WPr-nj` z_yD83`--E#|K^zsS2^daKiY_X**fI44?yw=YvJ^Hu4ddgD#ooEs8=JCaoV;D%!=gx zIWCx1!s6Ng&^u4R@c30|T4>$chnHS~{`@c`@D>y&$(Ugy6fjoW!(w0B8X>|=jMnAN z_;N(2C8^@@gV&q3|@e2BFu?M`=~ZA-F9 z(-@Y}8EdfGY`2XME7clIBizy$<8hfYWjFf)vAS-{^~apO>9|!r%~M-fX)6tK`j1Gx43YtCwxDpa(B-Od#B7zxAq`S; z0oBDG^#sGBVJyJJQgc!#+l2}mQiSSe)suK5uHKw=QbH@C$*39Y1Tu|76Yy$4TeSom z1|pTL^>I<|_qP2PoqN$!=cCdd>(*-beT*)8E+V6VBB?;y^n@(w8o^9R1R8u%M@g~J zOvGiLVC%%&auXpR9?z{4Td7i>VI4l7da7|i6uIT>|J--fLl;EQ6|rY=s%pc1{grpg zkw|$Bo*TTHZ_!}$>4|VlD3zM^NLrnsF2tUpH|1{BbXHxSnK0SM(ICj|W)#HCm zZ)%;@n4AI}OAzsmi2R{7IOqgmdYylA#c=(7AM}iP8-fU0~k(3LF2lTiXJ() zSX7fWR;^;_$*}gSug`&+o`0Wk^k)yex`U05^}0t^$j}gBy)RQ{*|&9{ zjoZ&#<)L@(_MjJw5INy&$V30R>{(R1ew+i7?*ypCLCc??dZl$zP23Cs9DnyC?>_kE zD>u5)=`MXI17WR8S9K6U1rUyE9;IG#-gY`P$5Y5lRPzUO`V^ zCZ>U;tX!1I!vb-_%ft{<@+ANa|JnWB$CsV*J9LUJdy|a*p$vKX$RC5LYmtJ@rj?q2 zKEv}iq=7%rl*@UF?~3^M8$8Ka>F#(1-Pc%1cMPV zSp&3GBQa?*+TEJPE*6X;hB(Pu72^arct>1y>4CdH`sX<`wB!HxkO-3&k(y3{9KD|& zaWWAj*WRnPqACGCH_Q8qbIr2SL87s;djPscicWHSXeg_d=5XYXmasSQN{cV-{H{Z0%*|?GWet#Tf=X)d~4^T7@XKFBvx>W9H zqSc0s5v8Y5N~tpB(O|*N^93nlP%KG{vxAhrTfm#787Jg;_nrI58#lc6QVQ+trR%up zj2(s?@el-rvJltRbGXJC{URq-)KttNu|A|IY*PYwzU9OYc)u=4m0CqGUo88$GR>jW z1k-&%NLuj2fxlPN<&ROk#C-UoiUV~&^ zF<>T9$4r(}9Hm-$WHHy>G>HvryspHGnMFv#Xidmt%pj+sfR%L22{)bmuNPNEbiYmI z&;vV&e0lG!;5_AB_>@L|kI&T#3|URZR}e*JSxlf7$4`*()FIPhgd8%pZ7cJ{GLpzD zFv){ydoJm<6w_L%C?@Jpuvxg&cRc^d*?X^l_0iQ@UB3Aax>nuD34TyAWU}~B=ax4{ z@*=sR+DK}0MUBPH^+GWmLr|dA+hc{&%__67@)E!S@CG~h$^&QJ^LiaU9iA~gtK9Gh z^;c+*S%~sSyI|f;Z5qEn2f2q{gwu&xZFZN3#hXS{%wobTVaqWceRf(Xm^2L`MrNk< z$HA^K78P*pzz2Kq){{;;{m~Ed=mfoU#{zBja3nAQB8iZliiE=|*U**Elfj{2q8#8m z*_ol7HX?j+W|ER(l0kE}Su^2Ctd!r{=X<6-C^Nh9)F=OQ_6dKxWcA(ONLDPp|3!)~ z!XDB#hrF~bS*sB1?3BqDBa3Y5zL3h8l`zF3DKXbOWZ`Op#?Z+Ou>v4I3r)S+nkoo| zifAMo5y8R@2zi zc*rZ)6X9r)yLld8uHWayYA#$X(P>nrK~Y>l$5dunbpZl>!nCLD`t0R^o)q-;92_n^ z6Kjr=a+0T9;sxy?0;GrTe&p#VPX71S?^l1oWv&X)(g^V+kOn7D#f^GK%nUmEZXYYA zW7uI&$h4Jg2!ugBssRWvajbP5JbbsEx_isD#~mj`JNfioe?UK8gHSKP0;!-YNt)XH zQK_S63nEUMFsFv{g=LoJw2oDle$Q4>4uDi}_ouDfd6XQfQ_hMy)b0Py{H z&;kzeZoU$%gY4m{w1e%=6ES^#thC^}x`c~e=%9X2=hln{enGDpabeu+bjD;rE~cG3$L z@Od-f8hbuG_Sx-w@87W+h;_POqV0x}-&Ib9L?DBn?p7=9*?8gWGRQ%FbT-unb1*k8 zib-S9Oz6=V!BFzy<-`g&T}5YXyg|;<gEpA*4L_j$j3e*A3m6-Tud;=k^SC%$v2+3gaw=+uDx!$k!*_c!c^7}wZ6LHJ z@LG0)GUd7Vytg{+4nH_LbRGZ5yBQck6B%upBQhFLYZn?vLGe*#pZNG8_p%t8w>$1} zTAQ4k_%&N#SdFUXJv^?jivZmAuM>A2^YPnvuR8Uor&k1E2<>(Q5rmqJVrDM(yhBY5 z!c&@$SC#X4)$Ovi`{;Y<_!Dow@)^{d zY~9*TKdjVJevAAr0ssn>KW8+=WJB-oN$ISv2q|~7I@7|U!L{9*G;K1BXadZ5#$uRF zEG3UY6N-t8w>j0qL_UHp2-TLMotGX`?emlB$euezXbe$rLA=y+$ zM=;e)ct!=QuETHP{5hFkAaseBO;*Pzh4|%|+a7%At|#wZO|t8-Jo@V(@~=27V~bJO zv6VgPtgt91k=U7PSzpWwDFZ>MovtpDJV8cSce1I|O2&&RVMe7TyXa=Q<|c(5=?+`{ zT*!~tx3q#dy!f}h@bTXtzx9ioSO0RvA6HtgKSo$2(1Z|BSOSESiHTT*LcZa_dWHCL zt#0AZ0>i|F-pqSuS%JbTXfS0?vs;Pa+I*_jIKKOeFE^d}>_4{y;Q(X|9*Cmd?jis4 zB7`yijL1VgQ&Cx=Jkz(Uw?n~{ZB+u(&K%%Zaq}MesmBCEYp4(^5 zs%MIWRkMHF-g}<7|AJL^1@|m)-|n@qkTD7(c&!C2XIIbQp$>8~oE#&pj+keOW0$JUfp){mkp{*>?S^^*HN2boGpoUmpNdASQ`wWpONIFUegH3kNMz zEh~=ZsGKH-*9%TU1#iL$s- zbRpgD^C14+`OiCFyXCY4hbSidPhM%*@R8eo{yUH?pbhZTFcUK}PNh`EJeJ4kl6Y*t z!7NL<7$_RXw{Y~4Mm`O;Alf^)>Et(G-~Q<;G^Nb|H$VS>o&zX_LEA=)%Uc3HkSJce`d<(2|7icJ zxKd15(FA^t{O0A06G-bpf|g2-FMZskKB96^X29rR;2vj zBS!^5{7r%IT|>K(s%^WOf?%%Z&pWI+Vj7&^z@DwDUlKLb35jh;2(mH5qKI9N%6O7m zZY#{`M6wdj_mkf~Wz&IetF+Wn=L^uY#ESg}EpT9PORvG!kXfDd zWe)LlOtoSHU?;?qqy_RIgkxpbO|1bTw(`7N+O zQdgVfB|dyqpRX8+v(7ALVZd3v>wzo&dE4#JcF<-{dh9&ahC3c!rG;h4i?_mM#K*bi zIRdvAp#)u`Xw^Trr?VraMoJcN^ZhuDi1Da|xZ*(NAJpe2T^@^sr4Vj!d+qFtZ~p8t z4|*5Nu9Og~yqf1w6ulmW}o3yE!+9q|l`;quIz`^#cBS z-KOi``{A5p(KDW6^>fMWsF%wmigJm5ypJNWl zw3KnybX(>^RgJx{j^FtB@r!qS`Q^n&t=f9v#R@u10_3X~u2EzKmP%THTIGrK6s@|T zZ<|FUQ`#l9IAoqtNTsPN(|LxmZ^^MBiO}yYiB$#;_RbrduQ~Rst3M^9oxAoGGRmE$ zBLxGT^mMnRG-D4+Bb)+{V8u3XoiE`E1^$TK#<0qRPKBl#XEGfDNl^pIlxTPfG1mQ4i( zJJ{;mcr=!xZ}QjO<1LRr@%Ilu{qDw}pdYpUgO!oG7rCipfTuPxh-3`u$++qi3sWJI zU)}8|RfBOd=E_qOl5ENzDmQZuE^Fd3Cx%dU<&?0Q(nYrj+v>$#M_>QyeHV?;R%<^y z7oEor^3WQjk)>_9uM(Rj~OUIlM~d8t42AGD(rPMat2mxQP})Y zB$MX);v^yDTu%Jx@AvQ9ce502^>j%BozXuafBbA4p!3F>f#85=b~t7=Ntj*zv`T~A zQkfI!CK+9&6=5$6LV#EZ7{r+nSAe7O!KSl5f9ZTY+Ejf1*JxA!izHuz`|7W=BvnZ% z=>4c`lPJl%pHQ+^X=jHF5w(pTE(A3B;)_xjlAEKQwkY5%+k&5K+ z4CdVANT`es`3{F4L-y+oIfC661sVh$dtTplGTbd3;^~wzPa<3Q;xm=w$&Eg=BS}7m5UrxfYh~s zUnCEk0V)QmH)L#Ybv* zxULV^KwAL@<+!iU-2C8!x2y(b{gz*#L+?Vqyy}mTl+Z}ahL(aS2n~mGOgL7hG|fX5 zE@&i+v%%KFT3aLnHI?5&mf=XazA+?VX{*tER;tYDmEmYYA9m0ddVCWuik)w~y6c}u zeSF{QPc~li5c(%{Wcvu-Gd9J{?@6a!CMT~+mgz*4sj%%p5NDub;&at~kw8zHYuO!; zX%_ZD?JkF@72V1}cZs-`1*tIwpSDQJo727`MenX4KfgXW019$8Y=U zlmkJu51)&lN4EhVdFb;!%B+K__4i0NCu63tm4y*q#+DT}20R1_C!1Ck64i*xk`5u9 z^A$#T{yg3BS)I07FT$H<`wCMGUsq1 z>P0h^jE%!|?Y)2h6Bpk6>}tBsJ7<*+IUKp<1PF7bp24)A4X0Q!n^5Lrg|t|?iLTYD z%t?-5I&m8Hg@DiIOD}l&T+ovhQBZAo_po1} zdys{AND}aGBW*}w{0*oS|)}y~r&gIYK%CKcBE~unbIc8!IlVfQ) zCIEujsjgo*ml2foDU^FpTDE$zr%|Qm0QkRn+xM^C_1p(vt!AxXkD<*ok)zya0Y2a7 zo4TZ0iIQ_`m&AF0QP#53tv0XWAJ~`urMv`mhGC%8WN_h7+jrZJ_b-2b(<&cvfA~DK z84g16gT?_~rl~HfVPe`&z=qWvO_`SyL`llv+=}tx++=FsnuKHa!OQnOwfoU~PG0pR zpIe~-U1UcJw126&tXQre%q&UZa^on~#=P1HR0%k)e!^g;0z$vS?HM$&JQ7?CFK*oN z&{LZ?KD^qF#lK}Bk$LaxV)#GE&SRmJg-fm_TZr1IwIox-p{3F&QG-3HH{4s!B1JNv z4L5FA?P?AOtO>a-EK@5G{_gwlyIXJn;g$e80f+x>RSS_rB0m7TLBW><1eJECkdz7C z{V6yzt4F+4US%sH!cNm>HW+Bbh?1UL2-)sbTdQf{1zq(ZjB$>s4hB$0-@f6w7tcQ* zl8Ecp{`0LLUDZA0l`G(r*_}46LU4#wvB}IMM50Eexe*IfoHeQ?DL}${qgdWpih||P z+@G7q+`!+u`16PNo^#p@7q7+-7yBv-Ushs9##=A>y)Wj9T4?UNIGdnS;eF{ zf!BMvQLa80EOQK#Bq2WTvnqh{?L6m%*Y17#B~$1xlEG=#c001HER&({jhcycJSHcms`X}G6R(;UogpqY%8BrbnJtrB zW&$3UBT@d_OR*~IMWsvl8mek%B4@S1IN*!B} z>r8fwp==1`5Jz5j`sa@xJm=K+R^2?J`Wv*H2=YHahDb^&kg)qCW1t|+Ng@FzSDcM9 zs4*y4nGl`hc^IGJ#D!{>%^&uWhW3!ZoWcW5)tpFcknRKvG1_(Ke{tCtpQEk=>(*W$ ztZKr%$bmE+l=p&`@u)+U5Cb9qjGYJDxc=DGlKLjSw9M@Tcx=lZSM2@Uo6q#n)8o6) z1*#24e{dZ7Q90zLt-n`fmGoALI~g&%M25l13goGUrno8ZFYug3+1E=6GlrbZPRuP_ zS(CF4XWO(M;<#|roT#10(jjKr_4p0DPCsy-3+>s7l_mPk90Umg4b}}h)lvxo+bT2q zCa!OmY2=meV#>0_M>W}wnqs018njG0;AJJH&1N&jC0I56OrMQQ*@Q?=HMiiMN?E>x zVREV(iZKd|<~M%0@uIK4xa{230NajSsR;fL5;_bhffmb>pXV2=K#LzVTIp6*<2Bm3 z7G7@PC&x)LY%Y)lm+~^zKe3cE(z(YJY&5fhjB{CYHB(8JZf4<0I1JMa&iCtHzVx%N zp1x_7Avx?|6#XI{(h)!@FgfYAjqoXw#HM0(xB)~nw)U!|x|9qh_Uw*GmSyKPasp~E zwe!l)Ucchy;}5X{WLs`Uug^8)mRo+R&U$Hh&<-fYkomlr>4~N^VxppVIYM4mG?uRFFUQq%a#@vXX z>ogk;<-+U4gt#(%x#XMFN{-CbmM_WSNxbK@dk(x6zxr*;50bgOryRSq@WLxTkFvpEuwNA{H%N z4m+QE;i~J8|JOUKel32nvURH=i}@Gu=xGorUByh^0jnaHkBjm6XpX@$(T3{JTBQL= z(hyh{rD*(RK+Xt+vH)P+zU!=Auipzqn{{j1J690A2ze$8d?}VN-JUo({Bl3Y_TsrI zWC*?=E%VT!&3WXI^D}7YDS0!Y%9jR$L<`bYkN)@Ay)S+Dz@e1V+x#xN_FBjX4?@O3 zTbP)_dE~Qazj+NM)qtSI!{4E^aU{a~`Nztv)9T?-kd9u|@!E(HX4ooW40B6g zp*N6#e!pOTaA6AuILo4Y_gs4WE9baZ?rSk%SOgsL0 z;|Xt{deI@Rf{TeG^shR|?O342(2~IsYf)kAs%72KSSMA9R)1_Tpp%C&rO!wvk>m^x zHRuYfg^+kUc-3pS?Rf8V^j)=XO>SMCTwLU58Nj}IQ^#OZk%Fdk+1e88?TL8~irXad zxggjP7+J!0CY5Z(ZHk&6TQw*1VnG)+_s)~HpK$yBgITn>TXZWrfd$$BG1NM8V%hdM z&o{ezJYR$r%Ehr>(_kERLJff`p=}tXvIubolEM^2j}h*ZLU6QR_r?BaH(q(jV*Tl# zkI`?mAje;^ga{#!r3%so#7eA;Wffy$8`w_!R7$;4n7}vFfegcmm5LR#k;2T# zx(05eYY7|-hzvHqd(OVC*FC>`HC*rf=oPd}KSv(FkPC4?cQMO}C?rxZUC=IB3_+95 zjrC0hvA?w-@msU7AV7l<;KA*?{`2H_yMckZZjCJbBf0|rh`jgJ;owsmH{+T~=F+c~ zI-tC-UKQuqCfc$?$g>zWJVq>if#9SFCf(x1(6L4#=LOaC{ z%W&g;w|;rt&PQ*6POe{juXl}V1M{%=(GxO*?EVY9Q2vmVjZ&GDy~j#NjCL=1qWyAS;1oDa_Z>+=j3dF7$8)UtBqsQtJ#)-tDwe`dEbgqu!%OUG7$De%32=@>QHan{pJmipK#*&tB*h9-Fs9UHXVLCI!#Gr{sGXuIC%}2 z8;IumCRrs^$t*KFWOfD~gW1odFTzfCcFL#XRTVy6%_&=CejG$v2fw|3|81|I^2};n zj%9y@ww6T_$rqK`%|B*_T7s#iNTDrjnqf>z=Nb|j60VgmPMH`QJma&(ph4fK3~+R; z+F9yRXaDikQx|@;GXK0RyI#G4@z+12Ls3WGAp#{+53XQxYHrzHMmx10 z)}j{+W>rbKlATR8w#GojoXwg2Jjg##Yi^6Z(;o39c{PueUIeG0OA)|P^m#bfu6ujW z4?mo9=^^&R=_jVp4>2IWdX)$vhzSFixKtyeQq7_uXE+_mS?aQv$ncj<#3Ft|NJUku zqF#^}&S%aSHk^emjTF?~!Yw%sLsJYLgU0Drmags(S0`|^W_N|%out{|iY z(h=t27PPx1q3OimpO9gfxWmf@{WC&o2b>VnQiJmhb&s7pXE zc3yM#`Hz48FlyVee(k0cexly+df^WA1J@xx+6}Olu9{ZsorpKCC)dPXPrXWy%KVaa zspuFAQ_IZE(;N%yAiFu5X(jGbL>A%F%0r4niDm3K*qxpytI#NhnoimTG1RA9095_^*}GPYq4Nt5`WZ50@4KhUvL0ih zxU4B-6g-14wA6ZbQKLpx$E~EMh0jTo3Zp0s!^m@p!G2827cYmT`V|sVwB@b;1U0)ayPz@7VJXynE;dIrq=MLjR40Y}q6Q<|UEl5j%w~!ZKsG z^A!zxCLtvXCmk+pY9@8mojEBzl*Nd(;4n=hRZ1SBIm_yGlpt8GrogfK<=xv(z44em ztIs(6%9UAg7P;W}DqzkhTTT$n^m*d#9Lo<1ye6wQ$>uI-C17vLW|AV@OIu#p`@=W8 zF43TaZU5kLv|AexEeEP%>AFR_$Mxb^?5W5x%1|71tfQ@uYa(u8Dl)?o903NwmaZ&P z(s5b1%Ux@aph+O?5V2%MJuHh})l(l3i z;dQ3043|6WBibkqted3)pG#GfKgBNeNVqPN$x8#iieUz`0-0|z)_SChc?hJ_v$DLI zNo&&z4R$?8lQ#uGTz>wvi(a|zjI$v%zHaR+(C>jigxuRfSkNj^l1g+9Ampu;TlwQ? zP8#ZZ0{IFfVVpZlr7kPRjONrHv7n(6J^$D5l04f zxd7L%N?l8z$cUfl`t(3l(DPtp0Dbc#IcZDpq3H#4ZJEMqH%y&gT1Z^>5_+6M78!Fa z#DPo#;qzA~z54zgpR81KWR*>d4M%NVA#+valZU`kiI!sW+KhaPFA%lT3t!&I!?+sM z7=f$CiGu-5OVbt`+Uc4|k$|Xd->&l?zxw_I9CSSP9J8XcQjy7BAhY%-vOdt|V+pt( zk&5xqoK&!cbhV1iZq5W^K^MhOkz2NnA@J39BGVo=Q)?=k zI&~=}XEb~kw!AgefDvk4+-Y~}CDxD~p-drZ_r}o|e*MuUTMpqy!z;9q7vCL2lVfaT z;{-|^!Sk9H*CSCf5u1r7RsLnjHurd!Uj1BXa};w(e`XHRGBGDlT*xqlfl1kH#uV{g z86Dz#?RZuq?6+QJouh zC`?;qI@4h2Morl=%wZNZLPZjDH5d^QTmW0Pp8Vh`7kqYz(DlwWThVy#N63Fp0osGq zJ_%41ZQU|nffsf>?+K;Jf+REQOeXBQg2PQPLP54JmSvWROs9+7_H^^|=CZm_!9(*@gB_m^{Y7uKXpE#mR`XW@Fx-i@nF`ApRWL$iy zSY7ZtjgT5`8C;T9U$QP|C^bgZq$Qm#SZ=D6j*Ctr5k zw};@z2lrag`6DBjJ^>qns1o@w5q%2Z3}l1DZaAovyLFi)0+k(jnjp(e*=rq_e7xhp zmZKj%=a>st$-l%Im!mBliBxq^bRlI=l?B;=o~vjOcq6C6xo%ApXwpqM8XZfK!K>5f z%V{W`!O&#YxfdQSZx~|wyD|+N%n#nM`}|wZI{}S6)~_8nU#Q;jAM;AxGly*Y&t7HL z7IT9Nr+G9J#jyh`(@+`^BIO1*%eGj=+@4jRq1S3-RU=Hbh*DE}!pfYIG06~CVX60| zb9Mog|5cTQ)^4RZ+;9tQJ^G~&FT8+(t`J{w#qrc4hrJ7L5<0%Rq={;l9G&XcF|2{; zIK%2@Yu#Gaoftv5lphh>&IQ3O*ZcS|nmhje!P`f@6++uNwY$1Z7D(;)m#MNxF8$pg zVn*xYd%Ofo#YT#dVyHagwzH1E|LR+x0@lN?*A&0{UbVsTE7l<~>m!g*h6*I0 z8VAvcqkf> zn_)lz&iLqD5VZhBbEXIH}I*Q`5}&_kn5FgPca3z&9)~`{qmN-opgu{ zkoQY5Bb=~X{(0fS_qLpVXk-8A^#{2e0mSM_hDuq z8vvRzEKq1EiZF#Wd6^Nur$!+zEprb}w_u7Gn4zoP038!NvQ!%}iEbSdxL<92a^ua% zY}~v$oPKlBo#;0lhTwl$13*!rr!u9+T%PXhP?NM_THVtq!O}j3n|E2k0JVdmWa-7U z-QiYx?GA<3H>1r`X-zCsRaNj8_ znk^k!3mQSQ$!Z(7IOB0$8Dvi;%n8oPuhQ9Edy$3BHgh6limS^r-0V>tqE4!0+*K~Z zd2dUAlPq~@HoO4HAio{9G`#TK-F5oCZ$5PH*+leDl0U(sALK(0ye= zqFP)Wq#4YZNuSe|ExCSPrmB%TOP&ain4wCy=xw*A4P0Q>a19}f-MRDGYhJqN!Y@`$ zUDR1+jl{@lU)%xLzYLKm6Rwm~;!h#YJT0u9wTIynDO1obLer8A9|AN>Pam+2~(|X<7Pd@FVx1}4odOOe$w9SH$ zSW`E9eO|n)&X9>w1%}!5P?}t9u@iJdRUgGyl{qR{XtKpf_ z0CLfvo^jdTr(b(3lpd~MtGx3k^@hNXDLR?INB*{HE3EV?(_1lE`rL7VQtxNId_{#Q zVDutOq(%{S!njN&O=+1$)TqS)S#Tgr+_C+dv)=mlv(?qM=j)X~TLU>c2G;}w+u}`F z!+?(w(I6_V*M#TCGBr}#R!fR>goHTM3^_y2 z5Hf(u&?z|xqQD>nd~2WI{R7T>_S@@y*7H2;ne|tknq1cD$HtokE$_ z60Sj2WNWiTt{ycxQ&X1#KHcgWr+ChpgrIN)yyZBUG*K#ZRZ-6&@&*(qBhc?cBI=Ga z{&LE*pK-BOv2pYkY|6~YAsnD|f>^~BXH4+gXkXXH@m)4x`-eR&C50R2ju6D?9=C*Q zr9sfpwVcY3*g2s93E<%F0~hRh>=>-3&X$e+!?gh^7wK$(1JO?^PM5_VtT3_@Z+FC7 za@f`JOq|KC{Fp7@Z}z_V&nHg0a})Xb?2L$Z8~%>sh#G_g!}TeQ*8xNx(j~ zY@DK8!(b>oLe{S<1E$5_|`{U);e!k>EQvJd}fO;ek$I@BMg$wRt=%b=;my)9gJnrBWS-3<`*zPH}^@T&W7T{9tW=Nyf#9|v-G z7Py_v!3ejMXd1|z7+v!1_{BiTmvEt6i>_-f;q`Gy{rbCi9RC(Z z%-FK=gVWaw@xKtxRwJg6bLqNHias%_4pYVnC7G}*#o%3@rnhmzrd+xZ(wp*(hb^{f zG8rp-C7l=O&d$(Z7~~+Fzw3dM9(@1W9MJdI^yt^tzdv=Y2vQ?li3HY8p2e%oxixA$ zuyWL^nqk3RYB2Kz>(WvkhRxh^S;|UKHNvD$JP(-ylXBZo4_dHUzk2`v2k(1$&BgoQ z$G?lo?>CU?&kR~LPz##}exXr|iuOrNSY9;+J>4djH{<&R(zohi}C!J@X>$V&Wi2Ujp=Ij;kaVv?}bCUC2@P zQY(j8#~wGvm1SF$G?tiJbHx|OoAL78pjnxi0O7ynt;6r0b?<{Ots6Y>=-i-0P<}YUs!BEeieFDykc1Qma93Kfpo^!p3Q)ghMj>(1=@2vQASbJ4 z8v|(fyFPj7v%BuN=lS*WyyB^&uwILiOJDx82EFd95fd9}a%S!plP48g=Hp?|Ivm;Z z)p>@WhAU4Ux(dwPOw)vW-nW zw~CU{sohQBsJmGsdzdjq@@jP}YOhY)0FxYf_WFn4e)>HPwxT~G{SjLZP>(nVNYl?n zB|^+ToRjTCw}qDu;)EQhgih+r7>V&Jo(5#+kuyFy@|PpKzJ;Avdu4j0BXy|FBfZdVix(uH?2S#$qov5zfN9cf*%j#p0uz#3lst@rF|UgISl%4 zxC30ohpyfE+q<9Nw03`JxpsK?8baFoZ@3`2c6F$%NP|J}pgbmv@_4S?#ZINho#240 zCpjmnLe4MZ);cMArLjOAN~GIbs>1pJeB^ZpE;|0@qd(ewi0!DXiS$24uDS-;Pde|C zp`0@XM%2?a+SOQtXt8OJr&c?ndKy_ z*(@-#=4iM%BAA^???OSffUh_#mV@W)-gEEun@Y*s;^6c8@C9bDxI&=GA0vFx=^!cf z>Xbp14UrSA__z%vcC@g^1{2^a%eZ-Z8=01E6c<#$B`m~~E~P#jPYMP`o??pd&^k#z z;R-8Mh>dn#_{lw&zqb1q*rK?=xmMa*kplQNqNGk3$4jr&ftgxTGr5D39MP*9XUSzE zrAS{+XG(*|7lKuF)+HAlcgim}eJjpn7O+kykqf{4J{YHWd7ToGQ!9u&yctWT=J{mo zm`oJJ>!;Si9G01S>?pT2`GEr8V^>yOI$uX}l$n}TvTdRlhZTw1m=eLY`P%ERpK{Ta z73>pbN3Zc(26EcD;M4Z=piCpxnWg%g22D()%dLm6IM#tl23baCfB#?<{CC=umpc2_bGQ~{76SwGgan4_vw~O_K zEuBz@L~36>NHPxbGGhkK;%y#MknZR=-QJ~qtgY3ivqA|z;^jRLyz}aD*KN`xAO7(R zSYSg!Y9AKB2brwN%FId@sTQ2ZDME(UOH_qwPN3u33d3W#hDa*77G5xqk|I8)Agb$W;g32zSAn&S+>@Q!}YU z3u7S78RJC`Q@O^3G~Ol8-TcVe&;DQ0?_Y5r!G>0bn7#llnP9F_MpAN5hDgjS<`gIn zSB(;q^R}T7HE~)l@MW#oRx(Rb&U5#dKAtVB1|`tt=REMMbANx$TFggVtT{t^1va;Emj z+HJ=^*1~p}0lDMfAYV2xYnF(|tT%J3xmLDajFTO$9K9~Atb9$ruh5S>l14K}MM{9D zcePlgbUU~Sm(i+fB1mKr;q~nU$jcl!>EH*)KelNBcO@-`4YvZh?3wGJX0x+w7HaE# z8v(?rI$youO>{{uZ-GkW+uZ|1k2iZmyvooEB}{?2k!TwcE_gs$d^4hk$1zxfrjae& zoHIbDw$s(p#`Ia?8eRKU%9Hf{360EAaDz zdhkfiPnr1moL~T0r5ZBTxW8o>uQ+=ZiaLBJ@Mksy%;G5JmJ+3 zXtrHW;c3^UoW{ueNf5~}hoTu%S)NS>{HUlkCpzb5)N6 z%$aW*=uUyzQfCymywv#m>@{9xc*XI86`Y& zs03P;IJuOSV+z2^j>o<{W6#@1wqf6JEiH?E0}0ustf11<9kh4+?qBY` zY2PWEA&w4WCTQ=6$jLwXn+E+J$3_!!f`XB}-euOgoE#K1*nY{QAMg0!m75=0ugznS z^Rc5ni;N8v8$PyhEx~1 z%@{&5>a&CI{q@C%HoXx4=i+~1?G2H4o`DAryEzDz(hcWCXDjiH#j%r@)%%6`Qefno z5rb4_X|@RaLdD_OY>v<2vApk?H(%X(;VvL=Z{0Xb{*-##Isdb^hV3Iq-AV!Tz5<_R zsnpqo7QbUN$Du+tTYtOh+J|mfchbCLE#dDY z-y6%|q^B-$dW4e$saHvEK@~HTKHY4Hl%9dXl;K-QrXU&e{CjrY_~h~TAA`M(wrrgF zhzA?hZzAVx-vA+tNCTUTLko;#G9#`>8+Efjr=p4vlpJ8!t!J?{B8L|!!@5S>ZJiKE zeh1s*f(3cU8~=Xb*_$X>=kQMhL?>de?|T{Ylm7vK3NEY2uN(Lw7K2=~Mk@?F7-gNOY{ZZew<4)l78v1Yes<-v zM-JZhE|4GGyMicxq6jlR z#h|%WU?o@+4tX}wK_u8@_MLK0Fwa^A38|`En&`GGqI{+6omv@2xq&}TyBrdrCLVd@ zix>C3^e+%rZP_^Rjs4h1*O7C#ybFw!D4R-Gn&1WIo&iNbPnc^+X~rXpIa zlhGVvXm~g5KX02Xd+r|%Pbry=2Ptt`{>fD^Jd-AmD^H*L-pX(m+pJ=uG8SE z+PZ=M8f1`y*T6@cgDDh^#p zpw06Ulr4`(P5Hu*)150hv|>1%R!_Zl$o&tqCYzT+j_DC4cc};seO{6pkDKgy@89nn zduT0%-m>wN!be!M3gjg?E>Vx#k`kikA@J#SZlpd$p$LdMYAh31lcvQCKEm^5B|YOz z2l7VUA0~8V7L71Wq)ewsTv&;EYB9atpCL+1h&RNlM&L3(_0XYXUwruuH8$**eX%ye z*oy4E0u=ei6*b4~4T(*>e#TQ*@j@QU%NKDYVX|k!HCUPA9Nv>3Ny5pbNMb^@QhnJ- z*1LwfwwFmZ&HK){q~L+maObnHzjMx0zgwR!SDw7C8%jm~AvjlqYA4}-7&jL5iqeQ> zsjpUQfN0cs{7ku=;w;-Uii66m#k9JSN~UF-+-|9Jh9;AAU&SPBN|i(ia&}#D=?#Cp z{Nzn|l>W2xuz}JcyY^zo>`=VqI<%!xphHr~6{Z}4rE%yZ28+p0OUjM*PTwEFyX#U_ zpT8XICc(uJZkHP$f9|ede)jz4o#~mcH4}5>>xz~dRmU|wh{+{Pgkuqy%cwyl9uUIZ zn1kY6G7DpWuGbn#IYWYmTnV=PU_&hrvXZ^2$6mDhG*jTpe0JMSPanKzojbnW@+kJD zDddH}fhU>2ppCShLYSv{BUzb@$AT`uHfT@Tt@27UA}2569&8MTTkD9$shmub8J>W(XSQd6gbS_-&8p^ zK;qVXwBPNIm~-jMWN_O(BEwQ{cE)o?F zzO(*-4MQENUKRqWyUv~%mi$?b&9-!T-NS)WF*LGk4xH9X zB9V-%P>;u|n$-z)nr@>(`bZ`jm|%Imc;728eDvE*diqbk{VLX=64?sYVW=s*N`y7; zq=IC#L1izdXPJ)XNqWPiC#W|$t0B|k4p*xsWt-SCrBl7ZkZHEqyJbX_%Au@mgGmmk zAvv-A#5eZe`O=Ml+O+#Sk8zSsA{DiqLr9S2COAEjl|`MQ4Ld* zB{8a5F1U5VGCl9v2XDONiCbU~9JO)$##pnB^>K4ks~kfzcY(U;r!@w65tD*An=mC1 z#q540y3ojKW2()!Z#=R}9OyY@R0@reHml|NK5uT~YCO#R6l~c|ycBHp0~h@H=1-o#bn_g!HU9&w zK_fz=e569HB}Iw_PgNY%p4rGEJBT!Yo};%Cdk$pPjH*o*}(mj!&Ja zQdJm_b$yaDZ%aESnz$pah0yKHzkYhdNq>B7lgM+Md#zq6AzywD#TO|IEg>y+P?tDr zbAd05LJpx>l-H5wRk3Q8h*xGFawDn+?CIbW&z}0b)Bdt46FGlR7`s#b$X(~Z4o+o= zk*Sa7>@`bq8k<#>lmUUmGMLp}_k5O=h%KRrJeTR%QVCyMNW>56M3q!q&G~3DvaFXZ zHG_POiIAlw#7S!1*Qt z3d^0w0sHil=k9swAJ2Tg_CM2Jw@wIr3-M#sqqrHmiy5Gcc>G+(-eP;kDQj*O`yD!V25m(i>q0U}$T2xH**R2a zLgoa;tOXXx1hIZhcBwNXr#i-@Q3FxeG{W`<#o{QhK~fOoApSaV(oJW6`RI;g)_eb% zty?wQcwb+?dcKByYZshsTFugcBcXj_pg%1c^fVkdkY(oFDsM7rMZ2__$fxJbL~KV# zFCeLU>T%Xr=*D@&P$$_vKC&lY3nZAq<^YL*Ln>utECM_SV6l+dQDFl}!VK|!<4f_rSMGr-0oYG($mO^KLQ>(JHpB;@xr;e>2ziz9#6;tn#UNW z&>AZ^oknraapBeKSXE~7*&4*!WcwSpY`=Hs4ac0h9+!j90*L<32iIYvoj^X^kyq1o!L}JFMUX?J+o~q{7F+!w+I#n>q^;>XPu$3vw?=C`9j1LW#waKd5xxU&WbX2xX$Twx>_S=9Lhs7 zanGSA{(Ri)r-Q-FmW{#V>r8JHsh{u&0Jg~hKHg3@rFcBwvWg3YX;ziq4i`wGWsL-d zV_Z>#Mkomu*xDqk%~^0Y1}~%BiODH&#$EK`p#vZNpK8_r{My@(kDxzBdw#mqt3St>Hah6A`D<3VB!od=DAT9fObs%&b!sbS?1<8A@2)pM0yWxZ~>sx4XJRmKw@C}<^^w&ptsUXYFmJBz(xG!S|Ku}4s!H-ECc`g@~`(j{pIiW z-nqW37GGbxmqTHv$nW5_Mxkm)Qn${pu?1x=Gl$MDBjzbS6(^06rk&F1_PVNxG~rX3 zNXfL`hH2^og|W;;vIB~I;?wV5f5omBHUWR`>?&*>RgrJZzYgn?CXHrIak{7*km~RQ z^HP>jYK`E3J&n7i9)Fqd?#_*r42d8T=K*lu{=peLK0Wlso=xELfmhdllp5sPy8wML zI$ScMha<|nN>x^;psSI4+zDYE3(J^nZYX6iEMr8Hm%T(w>G)VafPz#aD@qF|1N~gh zfG2ZXMk_4Ai>};%$tQn&7>AAg{tLOA55d9LF^x<2Ym>5$4 zf9=y?cVNlUrqy{cZ0Ywr`S5^EPRJb;zFt{OH3Y>)O?}NgMcu$0n<^|6lohGHb>vG(?Yq3RH=yC#Tns<1Z0wcV-#}{ z%^I4VA}0=PMDQsO<_V5a15%xUQ$PBz8$bELbXLJ;Zj?pS=x)!*P^R*zP5^nWU>Hr6&pyMJ)6$s&U+kUN4U&n!UawAr-DWT zw7>nHJ-__?lVjH6!QbL{0M#7V@F){X- z3k3~Qv#>=$(iqmyu}_`!%;`HGTYqKzwY8KggKT^S43mfwm!jo~EpQXEo0CE43JzZh zcDTllMdL1s(^@<~33u7cI}cpB{oG?6uN7Jc0|Le%v*tNH1j}EinpbJXMVE+= zkb`bb9X7M5BD1p8)fA!YO$nmv7({+|UUkD6FWvFs+Wl_lsEftdDr9fqr@*15g?-B+ zzvZ(ziy2_@GgpyiD2gTuL8nVHU?f#-GD}dhS(r0c%v@Pe1Piw?Mi~=Kv$|c8k0mv2 zdz&KT7$dx1+({SDAUHVT)nhNX=S&B-oCwc8sM+@WAFhD`JaPl|Nk~eJ2;woZES;^G z7K2w2(y#&Dpr>p?O;}NAcL#Qj$l4T+h`y06LUKxHtxVEx(mT}h7(VWH`xz|Bf~f4l zb#nGSkG}oePuKK?M_cRiUq3~5LM=!%W*cS6rZOe&B#CTZYJbQU^VMOZN#kgEgqhjI zC$vohDWOmv=;2*Mv^3zDivG#SVh$MMQF(9bW1=D|=$0q=F;TBgLv3b7SP8+%@Q z>f${|e*5{Rb?z}2d<$CuD@6L^Q^Ce4mV|RERN~VQ&X$3OBy;^1d%m zKK{Y=q7MCit$NEMzfQgdG=+HvbZ9cKV$4SCr8;R@ce8aWZGyA%tE#3>pwUTt9No6J z;*7E)11+WE=;0_yDH8wZ4vvZwF64YuMH(_I1z}1!)YfZs=K}UB{~=}MAR-g+7SmsJ*UZFJMcYTfWzUg1QTMNJmQq?1%F4= z=CbBu9LZxbXSF00w)=^{KK5^qU2>@j>xcUvYtzUq^76Bgx;B$&8Qq|;T$Ex3dQ{1R z>M7wdo9V3w3{ISgAfn8}4KcSVkw}T^VNOLOjr~?nz1^Bb`^|uS0XB^Yo8SR>`mrbe zaPOg0*BP_-|JhJ&bM0Q|{Y%Irm@gtZOK-#(61f#TYm0JLv5zQmd8q6d^$|Jb5WyG7 zx;O>+(3gL6{a-Hp>)T6g&+YlbtlifC?E3(+Vm_V?TW7TnTl>qcKH((}?P=%MRRg@41+1B3!xj1Db7z%XKZ8eW%v9QR9{O5ZSn8T4>Eoc|LPL1C`Y*q0f zNN)$y6rCbKOmmmAj)a)yQBpo9y=@>PW=3wZ{U68gKINp__Cda4%f{01eXK1ha_?S% zz*t-pF4UnDG7YNMP0lKMz*$a7{H2@{m+RYfAwtlU?Gjm9pv|RWB;UI8gk2B6amS{f zZOd!xOZh8E=zd7K1{so|PtZuE5VfprVN0BT^(3>hw(#-=!Q9RBt++y!UnjLnWg=AR z^+sHAp;s`~E^=1CL|m2DCd^JNXC$ZH9>EyY{G5t84A0dY-?{nb6Q8;gOr5^5anrRa z4ezV)?JDHEzXw|+G?4^3WL^p3V(9y!87lFd(Uy3Qh^_b*5CsZSv~FprfdMG2{$E1 zbxDNxw~&E2-*W4q7Dw2C_e3K(03F=|nVEoOW=>q2K}xSMuFgF#-17RT?{(@xD94aj3e;j(Z6x6{50Zaw_?^MAZ=y}llMdVT9Hk>Sq(XH~H1 zyy48E>J3E(d$`^3cE&T&7z*L>4a)LLhFZ-1xJ_qUE{%@4DqLHQ%J14G9m7E00|G&R$(0JY+oj~-?PKu~T zhK9IYXiCiCfX4Ljb?V9STJhgeWnDx3>5;GJ%#>Lk;Nd z$Di=QO~2o@9Wx=?vhi)&$Jl>pk#D~bst}ef(;j;b!jwH(xSRiW(6j_LpC{`nu!){=lp7XpxFdv9T*(L>%f z{|Deo5>ve9kha537Sj!NskJzTlD%M%{EdQ@-j&`%CiLh8du3y0HN?!*gepn98g(jpOhPZIg@zkgcTI}~McVm7A=X;)S18CczBU0Wk+4aKZr zzQc!e=Mq1kRN}2<%=Ia<(CF*)}A-uIr#_xY|1Q!90!kK zT4{z)Sp%#DW0}h>a_WA8M$GzK_G*#m_l+tomfRp%ctEe(OUL{pQ)gM>LCXKm-KU@M z)+7G}5)C*lC?ur-0X@+UA-nyF7^Umd|ly? zNJ$FmTx!tIr5g#O2EkR%UNqwP+P)iACq1y$H443Vq{o2B>FtnE~Mz@ely0s>52?Z|I)la|j=B>A%cK3Q8eKpa- zR(lBfLkwI&`GRyYXE8{)u8Les*Q)hnb6(GwaWpF~%?YPVNX?_645h7<2#I?loT6<9 z%;@QT=R9}k1J|va+kN{C8?J97Ti^aqPqAx{`f;*!*2UJMl#^krX$ zPnp!j)wILUMft9_k>&L*4Y4*u#|T9sjyds`v-a=5`jqwL{_XVjM^X{VZBl5}K&S)@ zgF zxgYP{G!^(J`aRXQa~bPoMicqPAvZkMlSQ(VCs>eMT7kAgW;-j3O5au}+PrDbg6e6N zqDf7PJu}g?xGulIoTbrGlQ>#dhq%<-N5n03UPqY?vH9UU&N;aErE4)0k*yo%;>DV6 zmBhN1odc;iApxfhLb(~#nD;7p*?KG1u_b-0K3PlfXXVD`m>iDSxOR0hBPaDCb`D3WO>Stu>?Wd;Vm2PB(1P)*FqdxzmtCi~#2`teOitV#O!Jcp zA^7L;g~fnbTXhc%xMYMLF!bDfkKJwK!hsP50k?dJW@QyXYp%Zc^Sdtp)gg$Ywrsq5 zniN|>N@V&6(1qD0wODr^O3N)EY3!i2$i!^o58b_Dv1e1nGZhyhBTa=%MDs+wszR=5 z;o>VaIz_iU?y~4CQ@XV1v-nOnp09`a@^3HQ^U?3W*sLD)z)wHIZ}?b){(#n8 z5*!Vy$<8V@yahBqm4y)S)7bakhlg%Ebnd31<)zC1z@98_q<9L6@ubn#@UN3ECU5Rj=(p;A4+1X=)c@>%ie20 z`u%zm_gqfG+J>@VrR^HDQq5QBD=K-BX&3znBg82LbRaP#8^Wf_m}7>rYStX5*|#=< z*Xe4R#1@=8)<6TLBw`LG)a;f#8RUsUQ`BMj>w;J;$m1HunTFZZ%#giSUEN3ZDB23q zY7qx~`^2j+zhm#7?_oo~;nQM6A45d9h#;|53KGd(gL5k8xFlqK$S@fD)ZmG)uE=IQ zvpN8Zd@2i|U+j$xZRx#WR!uYK^Eo1a;$_;~-`i_PxOk?s``9Z02dTOG^{ zdg6tx=#z^YOd`I~lH!^kHAkbI%ZXrwq7kd5S$0>cBbhAxEOE9p0D$@Yr4L>8+-bLC zxs_DUc%aya~sk4kOLFHZz~4EA-JKCBrMT3LqFKaOj+_ zAf(bpCB5ckffDzMQ(n9HvS&BFQQnmt!gO#s zYCs-l0SqwKP(;?Ug^wq2`bBnwLn$H>WoWfjPjMZBilk%PEQ3BsvW03D)u&Z2nK7GE ztd=yj)RC(k;ue<8qN+5Kr{sL%LOOOYQ~092UtzWd(Qv~Lbt^bg8XCh(MVcCcsE~!N zu}QnwDz?QCAz%8z#pj)LG?ducvT@X}q*_2AA%8IZn7~4 z?#+Fm=2tHZxSUew(78Dy1J_yZy0QaRo;t(jQ=WvvMxv&GMU12gTtveMa>WN{KY!{U zE`BA0_3i%A8g^kI&$a|wG==h6(kv}Ds;-vlPK!*Yw~FzZP}o2UY4d<46R7HAy0T#o z&g0c|)~IgDSe0?DSn@8k@wr`y&z5R>GKAqu8$YGDg;;V+i(S#^tPzrpIfp*HaQSO{ z-~a7z)_pK0U%}S-22x(=0osVsMni&htTebiVDL||3bmm)MQX4Zs(5#&L0wS?U;!tQ za7SHBr`z%%dbH9TcyhX;FCY`!9bvP_2dD9lXO92lj{Ubl5diS-Id*N1gCf`d4Bq`B zni&;!e4(Jy$YK*k^r*lZW1-85+tPL#)Jj}@0wyScK#Jze1(fvi^UY%%tb^l^|J`Lb z-gxz96uJ1yTAqwY{FE2eXv!&W;K+RO%HS7C?Xh}(Y~oMn!H?RDb*EMh-N0Cxw@#GkCMlQN?1g+i!C!^)Ql(jd9sxYZ7w2+e+f^92e10_ zjhpT)VO{y^(|2HFWkG(r8)&dB*|5M3EhQl23j*BTM6!sS6fH3I^fG^xBd59@B8byE z-rz)6f@5;$hYuZn(%JX?dOhklKe6W0agnEf2H8(dyAR|@uQpgSkn0VNNFd3H!V1=+ z%`PycOP@ezOjDW_cUhO04AeEW6lVqvu3!_4+?+V083OwqS0CARVDGD&TzQ}OWb7+{ zfLI`sL1k99%wn%hpcu3yO?E83OgpF02wM$9mCHCwWRq-)GN#YdW5MiP{a|&e zSf*jF|NhbSZ~XIy_t#VO;t$v7^ccB(nFKZ-g{ml{0g}IjPg=6-IfyOb zdL*>x{OnTBK}E|8_pWMA*q zl6XrSQ`HSgvi@L5?9eE^Y9K)!e*38xpLqPD^&&0*QI7o^5vd_D*i{_YA~o`dgo*qD z3a#j~24*;9%5qx`w$4Oi#(M*{JgpI>3M|dIEsNkg>=ZtpD1hq?r(NX%mVNZa_uqTg z;Y&An?EgD{El%4&cn`j$Mgt2T7uB=f9UG;h4U1VGsfE6ha<*VF|Maa)lSYv=5W zKC7ZK=NC`{1M0$>8Qq4*pyQVlYM=$jtVv~~!4mh}Jq68#RlquO;k~E7^yXRHu!*?l z{k1dQ7UXrupTVAwkW|)m8aj>XbGE$(BX6PaGZI{haVnC};tR9W=20(L?U@H9GC+-W z*RJgc-?@*AwKJ9-!6JmOAwN*T;@}tXmS%r3srO8y4vWNba~LZto@^nqEM`lT)YFaZ zp{ao5cYAY=sA$!+11<8v!GFAU@&l)Bo-J?gTl-?*k>kGlEzB+pTuTkWF87N7hMI!8 zkYW|}k2Q&Aixr)93ADIcneJ;_F4AD(N>UIqbE1L5Bu_iBC?ap43Z=7HTW{qFq7J}Y1&`C5HlpSX)KkG)HcM!hpZ zP%OlErK}X{D%344ErT~x)K?}nS|_Tr*$PFcTUwZ8KWHi0&5W8ysye3ywwNpA1d^iW z)H$D;O-`##L`tK5ayLwnCznA{akA&dPal2dpO3xyb?iSsfJ#vCx!*y)Xn{5@QjnkROluj}vJ9G?P)TfWN%o54aC0C^Qse~R3gR{A^Zm1V7TT6G9Voy`3`8b6% zT{hBo95=2&`4eqsSjOkve3=Dcv8!Ks{qd(yz7o?&ZrLaczk>bePmq89#RlM&KO9UU zx?(9m&ARj?V@RoNmnszvnZO#<0@M~q)JTy^lwdSI5j3?TCcEEFplP?QB6TPH&MEdX zJNfv3-~Hr`2iA?xu3TrG4M_U%LW8P;R&7&Uma*FrZ?S3UozS26`}lDBI1xb z$$^ff*Yt)dxopxFbvvB@bRFJAK$A7kKD>fS| z!0tB&r@=8R9%klL##z^Dgb3l|SC9MsAI`hszp3Jy$GH>VVHG@T%2-*#4ZzW;SYZh&5Cc+&WZKA3#!%__**`}dQc58JLyg_Cy zu3BnYGsI)2`pf0AL#T@N3H~&=1On8aFOL4xm;XF(O~$plZ^Qm_D>A(9N)YrT>>v~8 zspj-OkJb|6d%9E)ymD)JL|k{YTY9IfG( zM3vE}iajo|GBq)DN-5NawL=`f@A})GyYJBLn>@J3ckESfyD+(CCNhy*2|s~6*I1bI zHlx~^5Rw?-S}|W2@on;s&ohg*Og6ueh9fr9?%-5U;;?zmU`3LbkNf7amWm33lFBTQ z9ctlUcARj`hd1utu?|sxGFXEQ-$dTKstef&sk~)mqS~yr1tpK*Qlg@zxdNAu(UVSv z2kL1e#7sOK6d4VErj@5KQ6tG=OH7|h!xFhiO6K>V$`W%@WCh)Zcx9l%im3l)*I5@n zxA&Aa$8ah6zl-U+YKS&zX$$C1ZA7iSz*K8x|G%Z{j=QU@?@+~R%ByX)3ewl!z4zXG zzjyC-?%sRvdkqj9hEx(%pscWz(hv-@R*)gaFvAEm282Wrf+Qh?8H=>Pb6)?==aciE zd!FC(JiqZBt*%VQ>3Bfh!y_=rn+|DTuNCqYE}|FmpHn+`{9*gmm~+C*BNrN~ctoQD znIM)_X-~L<;ib51P?3400j26~xuo+IDH*Pcfmuq6#6{&|UEY@?0IN9m>tBC(@1EV5 z{MF=G1%1#zAnx&Db&hIh+eb==Vx_f5+RJ)5oJ$Lxp%zQGlum_FCxPv8>S#hKppQ@8 zbkC{V|8GYCjpnV}!M*WI>lnhegZz68(4!j8OhNNfYpPs27{k%WBoEcnwb*sOjkIw? zj*0{}p+nHk`7#B2Yh0w!H3qf>#O3p`N}>bws7K#D`P7laAHT8<+xQ~sKWWzc#4n(4 z>OxN3K?CtH!MIOBj00W<=ZhA?a@NQ?DUOFOyHE`E%q2UYKYifJ3jv6E?%Me`+^bpt z(bxYv*ry@^&Mt^j>X7eHdb*mfBv0dx8sdL>8+_zutY-{m;!_ff;UY9jRY#Ig<%=g1>r`#E zao8$L8JA<2EAz-K=AXFYk|ffQCbZN@W-?D0;le)j*4_JWz5nw~1Ev zb2_TLPD+u1Mj=U= zM|H?fI*6DV@u7xA&FlN(g#zH!lq;@G-YpNkLp`1k$Ngk0@E3d=LBty&QCzn6}(1VA!!f6dI@RwpT5*g%$Z(X3yU64`;JtmFWGj>V* z?YtPN#0D!bhG7di5;4e2Tu7gY^7UFWXoWw3={12z@j~%NaTUyP-TD`^vZ3AAK7;E z^P9bJ4h_>@q#?(4LzCZ@_1PK0bK@c;juK0u%pj+t1G&y4LKnMun3PZri+TH@j}qf>uq3HTjpwN zttfPSz0*lTYi|urMjfBSDWv$F1U2l0F!=D{cVF1{-m92hsZoGQJ>!t1Wlfvg80uCw z<`j_lD}B#A*dWau%NQ=o1ilB=Lvp2T)WIZZnu;a3DK6b!iUka-w4opT>jy(27IF;Cy*}#-ov_&>uLPwM<4Tfq} zHxd-YqKYT#B)jsJz-YOY1!V5LfeQrxN8i5cwp$*$5$Gl7tO>q~fn`$2h}wbXiBSxa z{KK#$#p^jGB|~o%cI(w9hQ!+uwu8eIyON@{`QVqLD{H8kI`p)5J@%`Ic3gHnX7$3{ zk72Ya$Se2#k0z&FvP_93gRGM0ikXTJCekTi7C-6WLli==5S6s0t~A~V&3R_jIEzt2 zp&2c$cEv3ur4e1LTmEh9%R5eE`Puh8ccBd)9U|4kcutqu7}}JbyfzqwHeP68aeJLH zP+K)5DT%^r1_dBK>&YeM1xcAv$&glV^E_fr45wvZpe_=xbkK_Y*{@!=m}D|Ox%G5Qe&A)W{Q2LaFR_Gv3fl+DNE z66(ar8lfxd#;!aYF65U<3uvQd3aOAb5Lw~Ks&QRng8tMU`ySbS%{`xC{D}j<`7*jg zImjdNyH&XhmW`ySRVGd2o~q4APK=~o85E6%`n(ER{P7!)Jb3iu3(<5GtZ2{NqFMh& z6_Z0KA-MNI`9~>=NBUHIG3-g2Jc@px6mt<4iiU=7??}muP-2uF3Mf%XYG&L6KK<>fv~nh zdf<{tBr|GHE%cLOR$oHY@(M~uK`K(~t66rtBNvI-dW{@|9ApLEoKDC|Y&HyerOO#Z zHYk;1ze3aDTlBJpX=d{!NoZ?5^!#7;y?*hD&C%}zFSgMSN=4#CXql<$@dRz9V$9`S zDM{Iq)hvxx(V(zq=OKoqWf%F%Hsgh|Y>|mr`Su1$(4Y7__>_uGad8&{ z1Ai5g6NxNu-jU#PjB5JCG~4j{B9pDBpZoobmwoa>)G*?lwg0YQDiVK>T)Z7ZR-Fz= zB8sZUY>o4(c#2U>69IayJrC6IrB2#vA`f{Rb7xm)YAuFtAiSE{O3zFmpSB~px{L_4 z>D%{z^5R{OZ&IG_xf;{oI}6#f71()6Mz0OHYH{C{0aw!+kE+WXNA|o@oG@0&G@V{L zkRe5St>{-!`;+C6F-q{W6+OLfiNu%073hR&^CS`5z6dQlNqt){Ey}z05fUBLH>4a$ zyLX*>>$4j!dT^7(`GgiTNiZUroyXNVzOodPcLo){kX7?+*h3PdtimevU8Sb1GE}-N zO0|zdD=h{_x-{o&n6!9M4ZZTyU%qt3W1kGrPc-v?usIPve0u{Fn4rZc?*XvAJIotA z{Hf9DHXF?Z7jHJPO-f>;QDhjgQmvj(C526V-%sw^bHlxV!U&ejuP)IT`37>f@;c~6 z3e1SeS^#<{I7v?kDO1v~W3v^_LNd{(uxl(qJs}RzaWjP2@&l9Q_a`p7;fE(LK;P$~ zzdMX>6Fm}sqpi&m+{s?7iQu{8@Hkv=GtAK%(Wk026$ucSQkP;%CO|lzICj&%+n)G* zQ*voPZMT#aPSWl4Ou+{Ih|qE;6|4 zElP1|tQWN8GTL!^@bT9k{P|UwTg`(XCeXJLAd^EyFe~Yab%5G7ddESz7~hq-0%D8Q z&rVce{IHC>A#qumLp{MLvJq_9>EXPqp%4oi9AhySSr&ulp&gl}3C>(huWr^CT0uI_ zsXz!v6-)VVUJ{o6G^Ge~PTJaGL%V*KE)8tc8k| zf$*$0A*lk*?;yxk+2fG*Y`^@9_wPG&486LytiARW%)q~53!)MA1!Vt4KrQB{W-M-= zOVM`Wn$YhG3oZ8L8u6-(CP5l>r@hTv8o~BTC7GxK*taym`d*@NOrDwX)@fr*3{-rxh*4>5S#3?TQ8NP@w@yK(HA2zw z{Li0x?y_h9fSN);K5*gr=$_FdWl2t(OKGN7D~I1E1#vc!CmXNQ(-A7gAqi8}3b2Pr zYpZbi;>{9)Yh=ka5~6TE>XdHSo4s6H*B+WjJT2Xh=dmj}n>orQ`Ku0kt{50w`?^dL zL{p$(-hI;*??3tHcQ9Rh#r=EGKYbSRG`}djE|-f8oUspR}Ow9E#w<==Yod zg08Ix+5R;E**Ha#VSX4Q)*F=BD4eC}WO-Mm&#{HF;sjz7N}Q8u2(QW2HB5PoTO~`= zF1H49u*hh=oJNF&j^U^(e zG?Kpe8fE~LK-5Y=GmB>4P|h9|3N-Q#VU=;Ht(K`)1N>*zfF0`aODg$1OlaVMkLwoL z$+ILc8aBb4^P0yVz5C@$w|$0fo$p@`!g`hS%jgIBI`X?caH!KQ=D}8H$!A>Z{UfTC-1(W`g1f2N~7{?E!N`*Vod6IT;C3)H`0>zUS!9hr##( zRDKn9?fRMfFp)MYvVPW$Fe5himYkS?mtO@7LnPeqdn7d5yuMh;c@;stWmk5z-mx&F zoG225nkhxja0LK$J@wRWTR(Z_HVA-o*8XxcX5`pGu74JQg~ZrkHL%;3bS|h$swwky zYsO`R3V$rlpb@O`A$7JKPL8~A?T>d{@$6n`z-(E2d~Z^-e#DP0X#;t+bT**rRsKR| z7AwbvK68{*2Bk=oT%((1gF-!B!<%jp=qYFv9r?rQhu*&7ub8caU&Q=}49L-+LNZ3g z8ChF=K zpLIanP&?ENbc0*+@j~@JUyE)rjk<3_9VF5jQH1d>oR^6LNt9 zFC6AC9=!U#t8af8Tl_yik2wo`0lCu#--ONqud8u^t_A7z9#58vI(hR9nQNhy?DKv;^ zyX(f=J;7)R*J{t%Wd5w*Gc3Xkk~^at#=GJI428Qy?lJ>gh}YEE^K-K=>j17boflIY?D5!=S4wz> zDKB5D?l5w^B8N>K%S}TMjo=0B->3U`-~GyO-o)4irf<^G)i@t{`#3->n0V6yT*!o! zBHklhFf`^>pDog7wK2KG?*!h3pA>BsihedU<@u0V9ld(nuFHRkUg=xbUg6}m>(k!? z$_5oA%^rbMhm!{ciEnApk_|?kqL{I|79688mU5QrN}ZQd2!XS=d~U~4B%YL!QwPhA z$M#?K!VjMK3c6)(`;7%%4F(c4n4rR`jf0#l-Pt!3vV_(OX(aLs>jr5~W7k3YdRnC8 zO3FGu)JrvJRHaFnNf-27y?CrH0zC!x_wJjv{_^*K#@GoTJ&Hr0`%R>IE|`m|tjj2# zn(x#oZFStR$mFLCMQC0%_Gq?XCm33I6TXzHCWy{@A-Z?0o>mlPO06G@C1ZsNz$}jc z_|en9f8fH+x%v;p7ofGyI?_o4J?#8mW7$y~$0Z*ykkh?c)m0BQJc0y%JbCnzgBRZQ zt6f(BI&aHb_X3`F{SQxXM=wkq;>|!lNo;hrF@&L^1C~!C3yy?~Qia1aIBcslZ!uA2 zyY2`~nXi0p>(1XE`CbF<#2q;Hef9cJblcPzxvPd8xa@7fEP~A9oIjy5@}#MXUIB}b z*kY>ZvPH{QhCpGB{Nr35U*Bk!XkC%46g0JHcJ78`NEZq1OTCN)qQe%Gd2Wl_VP&>m z_mlVj^Uz*Qb?R&HM$tvqAm!sw4Fv{Ho1-&L%pEF1EYj4woGwx`$<*;yy>zpfUXfyQ z^&&K~GOV#ILtit+<#SEBkx^G!vW=wzNFb^OPVF&`LzIbyU=ufmE;2f*Km7RK*AM;( z|Qz+dVS$y^R{fvTSe;`AiF zk@8t>a64rR;h+i02qsDibFDkS?%5wS(wWt!#SNg4%kvqZ(7WNsbM^SDFmQr`dzZ!DoVuwo?%IE!B#^ljM zCtrT~iI4XD4f?8Ic^^Bc!U$UdxOK8r#IVkj%0{M3sA)u0a32#6v%x~5FV*QvJcmJ5 zlsaut{{8N!mmYoRVH8b%&e~Ncu$HnBc~39{YKOIw&{R!s$5KI!*IHS(+n-m?$X-jz z6p`g;#=J%T-_d?A*EagaCpS~!z^qtIds`T_)lOq5pE zeJu)sHthz7Ccdp>C#&Pc(L4*8qp)fpjQoRSld7daY&25(RJ5Eh!l{4gjc0ei`j-<} zlZzErpqtKzXb?d7=E_<+olkA4z4N5e6MzJHnJ8BnYHgV$4qFnF-XPz1FC_gKajwkf z6*Lh(mRkV-5g}A{?2dHQCsy6HG_&4g5c2hAJno#~L-onbu~LeifCj)4A*zd?xae--kdF=M(BimMt7v9RN1$p?13OW-%) zz!FWKZIPm&M5n9QrxvoY4FZ>v@YoMfMPdWd@xK25w`vNFk*$u106`^5k}|B$9_jG;Q3}V>?7F zpRtrz0TzY?>PIF!=F6ojUjI{|76<(Oduk literal 0 HcmV?d00001 diff --git a/tests/google_messages.proto b/tests/google_messages.proto new file mode 100644 index 00000000000..489c4707327 --- /dev/null +++ b/tests/google_messages.proto @@ -0,0 +1,149 @@ + +package benchmarks; + +option optimize_for = SPEED; + +enum Foo { + FOO_VALUE = 1; + FOO_VALUE2 = 2; +} + +message Simple { + message M2 { + optional int32 f1 = 1234567; + } + optional M2 m2 = 1; +} + +message SpeedMessage1 { + required string field1 = 1; + optional string field9 = 9; + optional string field18 = 18; + optional bool field80 = 80 [default=false]; + optional bool field81 = 81 [default=true]; + required int32 field2 = 2; + required int32 field3 = 3; + optional int32 field280 = 280; + optional int32 field6 = 6 [default=0]; + optional int64 field22 = 22; + optional string field4 = 4; + repeated fixed64 field5 = 5; + optional bool field59 = 59 [default=false]; + optional string field7 = 7; + optional int32 field16 = 16; + optional int32 field130 = 130 [default=0]; + optional bool field12 = 12 [default=true]; + optional bool field17 = 17 [default=true]; + optional bool field13 = 13 [default=true]; + optional bool field14 = 14 [default=true]; + optional int32 field104 = 104 [default=0]; + optional int32 field100 = 100 [default=0]; + optional int32 field101 = 101 [default=0]; + optional string field102 = 102; + optional string field103 = 103; + optional int32 field29 = 29 [default=0]; + optional bool field30 = 30 [default=false]; + optional int32 field60 = 60 [default=-1]; + optional int32 field271 = 271 [default=-1]; + optional int32 field272 = 272 [default=-1]; + optional int32 field150 = 150; + optional int32 field23 = 23 [default=0]; + optional bool field24 = 24 [default=false]; + optional int32 field25 = 25 [default=0]; + optional SpeedMessage1SubMessage field15 = 15; + optional bool field78 = 78; + optional int32 field67 = 67 [default=0]; + optional int32 field68 = 68; + optional int32 field128 = 128 [default=0]; + optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"]; + optional int32 field131 = 131 [default=0]; + optional Foo field132 = 132 [default=FOO_VALUE]; +} + +message SpeedMessage1SubMessage { + optional int32 field1 = 1 [default=0]; + optional int32 field2 = 2 [default=0]; + optional int32 field3 = 3 [default=0]; + optional string field15 = 15 [default="FOOBAR!"]; + optional bool field12 = 12 [default=true]; + optional int64 field13 = 13; + optional int64 field14 = 14; + optional int32 field16 = 16; + optional int32 field19 = 19 [default=2]; + optional bool field20 = 20 [default=true]; + optional bool field28 = 28 [default=true]; + optional fixed64 field21 = 21; + optional int32 field22 = 22; + optional bool field23 = 23 [ default=false ]; + optional bool field206 = 206 [default=false]; + optional fixed32 field203 = 203; + optional int32 field204 = 204; + optional string field205 = 205; + optional uint64 field207 = 207; + optional uint64 field300 = 300; +} + +message SpeedMessage2 { + optional string field1 = 1; + optional int64 field3 = 3; + optional int64 field4 = 4; + optional int64 field30 = 30; + optional bool field75 = 75 [default=false]; + optional string field6 = 6; + optional bytes field2 = 2; + optional int32 field21 = 21 [default=0]; + optional int32 field71 = 71; + optional float field25 = 25; + optional int32 field109 = 109 [default=0]; + optional int32 field210 = 210 [default=0]; + optional int32 field211 = 211 [default=0]; + optional int32 field212 = 212 [default=0]; + optional int32 field213 = 213 [default=0]; + optional int32 field216 = 216 [default=0]; + optional int32 field217 = 217 [default=0]; + optional int32 field218 = 218 [default=0]; + optional int32 field220 = 220 [default=0]; + optional int32 field221 = 221 [default=0]; + optional float field222 = 222 [default=0.0]; + optional int32 field63 = 63; + + repeated group Group1 = 10 { + required float field11 = 11; + optional float field26 = 26; + optional string field12 = 12; + optional string field13 = 13; + repeated string field14 = 14; + required uint64 field15 = 15; + optional int32 field5 = 5; + optional string field27 = 27; + optional int32 field28 = 28; + optional string field29 = 29; + optional string field16 = 16; + repeated string field22 = 22; + repeated int32 field73 = 73; + optional int32 field20 = 20 [default=0]; + optional string field24 = 24; + optional SpeedMessage2GroupedMessage field31 = 31; + } + repeated string field128 = 128; + optional int64 field131 = 131; + repeated string field127 = 127; + optional int32 field129 = 129; + repeated int64 field130 = 130; + optional bool field205 = 205 [default=false]; + optional bool field206 = 206 [default=false]; +} + +message SpeedMessage2GroupedMessage { + optional float field1 = 1; + optional float field2 = 2; + optional float field3 = 3 [default=0.0]; + optional bool field4 = 4; + optional bool field5 = 5; + optional bool field6 = 6 [default=true]; + optional bool field7 = 7 [default=false]; + optional float field8 = 8; + optional bool field9 = 9; + optional float field10 = 10; + optional int64 field11 = 11; +} diff --git a/tests/json/enum_from_separate_file.proto b/tests/json/enum_from_separate_file.proto new file mode 100644 index 00000000000..ceb9b42aa1c --- /dev/null +++ b/tests/json/enum_from_separate_file.proto @@ -0,0 +1,9 @@ +syntax = "proto2"; + +import "tests/json/test.proto"; + +package upb.test.json; + +message ImportEnum { + optional MyEnum e = 1; +} diff --git a/tests/json/test.proto b/tests/json/test.proto new file mode 100644 index 00000000000..2db0388d778 --- /dev/null +++ b/tests/json/test.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package upb.test.json; + +message TestMessage { + int32 optional_int32 = 1; + int64 optional_int64 = 2; + int32 optional_uint32 = 3; + int64 optional_uint64 = 4; + string optional_string = 5; + bytes optional_bytes = 6; + bool optional_bool = 7; + SubMessage optional_msg = 8; + MyEnum optional_enum = 9; + + repeated int32 repeated_int32 = 11; + repeated int64 repeated_int64 = 12; + repeated uint32 repeated_uint32 = 13; + repeated uint64 repeated_uint64 = 14; + repeated string repeated_string = 15; + repeated bytes repeated_bytes = 16; + repeated bool repeated_bool = 17; + repeated SubMessage repeated_msg = 18; + repeated MyEnum repeated_enum = 19; + + map map_string_string = 20; + map map_int32_string = 21; + map map_bool_string = 22; + map map_string_int32 = 23; + map map_string_bool = 24; + map map_string_msg = 25; + + oneof o { + int32 oneof_int32 = 26; + int64 oneof_int64 = 27; + } +} + +message SubMessage { + int32 foo = 1; +} + +enum MyEnum { + A = 0; + B = 1; + C = 2; +} diff --git a/tests/json/test.proto.pb b/tests/json/test.proto.pb new file mode 100644 index 0000000000000000000000000000000000000000..94b8b92e61664fd0d5d9ddef9b2c88668a7fb951 GIT binary patch literal 1958 zcma)-U60a06o$8SyTakC%z}0?nstrQ7%d4)h}n4IVm8qWZn|z_^r9)TOHk-Ew3zUF z{0aU+&&>4n16J4@+IjoVGiT1*De$KWo!F1!sP{ACK~LP;;gZLk)mP!fmSgec>~90A zU-0SJk0NjCvlr0dVLao3H+N@2eBQUTQ|*=E)Jgp;#7*09V3p9O7i_};dkIaN=dKh^ zm$-D18>wy?5qKzp1>*)WikGusY8hy(IH~bks)Y(CYuvDPguYr<;Yf(=Iix#tC!Ei% z2k0t0H7a|A8ha1v*j+?ZtBjs%|HLk|(!O3zk}f(`nm&%E>>{^>AFLKu1*bRqosH?^ z^%XwMRu&_%i=`j>UhMx!;;5eLA`Vj1I7kpS>|PCR7ze3nQ_UlR;;5sq?xc%kaTJUQ zY@qS5XxtzsaWv70anSrM4pLkdhlxIngVZz*5>y;5bYUDMQXDjvaWM4kj)Uw;95i%g z9LG5Q|8bC4#_=6m3omrlPLqq(mdp1x;>;Z?d+dd(4eIwQh?i@piKM*BzJe*>@<0%~ zBiZ(DwtNTj=mv6fBD(>xm~1G4A4>4s-C)t}JhzUNOrwG$dEk=Q^U-|@R!mu;>V+sc zCsM%)c_PlGb)SM2LzJkvqTozp4v2*)K5>#yB=ybiHYftVI|^D$q2hIV22~#T{5IKt z7tglw>A(Rg$}JWwc#j LrOA=}D?a}R1Rqz2 literal 0 HcmV?d00001 diff --git a/tests/json/test_json.cc b/tests/json/test_json.cc new file mode 100644 index 00000000000..66508179b3f --- /dev/null +++ b/tests/json/test_json.cc @@ -0,0 +1,256 @@ +/* + * + * A set of tests for JSON parsing and serialization. + */ + +#include "tests/json/test.upbdefs.h" +#include "tests/json/test.upb.h" // Test that it compiles for C++. +#include "tests/test_util.h" +#include "tests/upb_test.h" +#include "upb/handlers.h" +#include "upb/json/parser.h" +#include "upb/json/printer.h" +#include "upb/upb.h" + +#include + +#include "upb/port_def.inc" + +// Macros for readability in test case list: allows us to give TEST("...") / +// EXPECT("...") pairs. +#define TEST(x) x +#define EXPECT_SAME NULL +#define EXPECT(x) x +#define TEST_SENTINEL { NULL, NULL } + +struct TestCase { + const char* input; + const char* expected; +}; + +bool verbose = false; + +static TestCase kTestRoundtripMessages[] = { + // Test most fields here. + { + TEST("{\"optionalInt32\":-42,\"optionalString\":\"Test\\u0001Message\"," + "\"optionalMsg\":{\"foo\":42}," + "\"optionalBool\":true,\"repeatedMsg\":[{\"foo\":1}," + "{\"foo\":2}]}"), + EXPECT_SAME + }, + // We must also recognize raw proto names. + { + TEST("{\"optional_int32\":-42,\"optional_string\":\"Test\\u0001Message\"," + "\"optional_msg\":{\"foo\":42}," + "\"optional_bool\":true,\"repeated_msg\":[{\"foo\":1}," + "{\"foo\":2}]}"), + EXPECT("{\"optionalInt32\":-42,\"optionalString\":\"Test\\u0001Message\"," + "\"optionalMsg\":{\"foo\":42}," + "\"optionalBool\":true,\"repeatedMsg\":[{\"foo\":1}," + "{\"foo\":2}]}") + }, + // Test special escapes in strings. + { + TEST("{\"repeatedString\":[\"\\b\",\"\\r\",\"\\n\",\"\\f\",\"\\t\"," + "\"\uFFFF\"]}"), + EXPECT_SAME + }, + // Test enum symbolic names. + { + // The common case: parse and print the symbolic name. + TEST("{\"optionalEnum\":\"A\"}"), + EXPECT_SAME + }, + { + // Unknown enum value: will be printed as an integer. + TEST("{\"optionalEnum\":42}"), + EXPECT_SAME + }, + { + // Known enum value: we're happy to parse an integer but we will re-emit the + // symbolic name. + TEST("{\"optionalEnum\":1}"), + EXPECT("{\"optionalEnum\":\"B\"}") + }, + // UTF-8 tests: escapes -> literal UTF8 in output. + { + // Note double escape on \uXXXX: we want the escape to be processed by the + // JSON parser, not by the C++ compiler! + TEST("{\"optionalString\":\"\\u007F\"}"), + EXPECT("{\"optionalString\":\"\x7F\"}") + }, + { + TEST("{\"optionalString\":\"\\u0080\"}"), + EXPECT("{\"optionalString\":\"\xC2\x80\"}") + }, + { + TEST("{\"optionalString\":\"\\u07FF\"}"), + EXPECT("{\"optionalString\":\"\xDF\xBF\"}") + }, + { + TEST("{\"optionalString\":\"\\u0800\"}"), + EXPECT("{\"optionalString\":\"\xE0\xA0\x80\"}") + }, + { + TEST("{\"optionalString\":\"\\uFFFF\"}"), + EXPECT("{\"optionalString\":\"\xEF\xBF\xBF\"}") + }, + // map-field tests + { + TEST("{\"mapStringString\":{\"a\":\"value1\",\"b\":\"value2\"," + "\"c\":\"value3\"}}"), + EXPECT_SAME + }, + { + TEST("{\"mapInt32String\":{\"1\":\"value1\",\"-1\":\"value2\"," + "\"1234\":\"value3\"}}"), + EXPECT_SAME + }, + { + TEST("{\"mapBoolString\":{\"false\":\"value1\",\"true\":\"value2\"}}"), + EXPECT_SAME + }, + { + TEST("{\"mapStringInt32\":{\"asdf\":1234,\"jkl;\":-1}}"), + EXPECT_SAME + }, + { + TEST("{\"mapStringBool\":{\"asdf\":true,\"jkl;\":false}}"), + EXPECT_SAME + }, + { + TEST("{\"mapStringMsg\":{\"asdf\":{\"foo\":42},\"jkl;\":{\"foo\":84}}}"), + EXPECT_SAME + }, + TEST_SENTINEL +}; + +static TestCase kTestRoundtripMessagesPreserve[] = { + // Test most fields here. + { + TEST("{\"optional_int32\":-42,\"optional_string\":\"Test\\u0001Message\"," + "\"optional_msg\":{\"foo\":42}," + "\"optional_bool\":true,\"repeated_msg\":[{\"foo\":1}," + "{\"foo\":2}]}"), + EXPECT_SAME + }, + TEST_SENTINEL +}; + +class StringSink { + public: + StringSink() { + upb_byteshandler_init(&byteshandler_); + upb_byteshandler_setstring(&byteshandler_, &str_handler, NULL); + upb_bytessink_reset(&bytessink_, &byteshandler_, &s_); + } + ~StringSink() { } + + upb_bytessink Sink() { return bytessink_; } + + const std::string& Data() { return s_; } + + private: + + static size_t str_handler(void* _closure, const void* hd, + const char* data, size_t len, + const upb_bufhandle* handle) { + UPB_UNUSED(hd); + UPB_UNUSED(handle); + std::string* s = static_cast(_closure); + std::string appended(data, len); + s->append(data, len); + return len; + } + + upb_byteshandler byteshandler_; + upb_bytessink bytessink_; + std::string s_; +}; + +void test_json_roundtrip_message(const char* json_src, + const char* json_expected, + const upb::Handlers* serialize_handlers, + const upb::json::ParserMethodPtr parser_method, + int seam) { + VerboseParserEnvironment env(verbose); + StringSink data_sink; + upb::json::PrinterPtr printer = upb::json::PrinterPtr::Create( + env.arena(), serialize_handlers, data_sink.Sink()); + upb::json::ParserPtr parser = upb::json::ParserPtr::Create( + env.arena(), parser_method, NULL, printer.input(), env.status(), false); + env.ResetBytesSink(parser.input()); + env.Reset(json_src, strlen(json_src), false, false); + + bool ok = env.Start() && + env.ParseBuffer(seam) && + env.ParseBuffer(-1) && + env.End(); + + ASSERT(ok); + ASSERT(env.CheckConsistency()); + + if (memcmp(json_expected, + data_sink.Data().data(), + data_sink.Data().size())) { + fprintf(stderr, + "JSON parse/serialize roundtrip result differs:\n" + "Original:\n%s\nParsed/Serialized:\n%s\n", + json_src, data_sink.Data().c_str()); + abort(); + } +} + +// Starts with a message in JSON format, parses and directly serializes again, +// and compares the result. +void test_json_roundtrip() { + upb::SymbolTable symtab; + upb::HandlerCache serialize_handlercache( + upb::json::PrinterPtr::NewCache(false)); + upb::json::CodeCache parse_codecache; + + upb::MessageDefPtr md(upb_test_json_TestMessage_getmsgdef(symtab.ptr())); + ASSERT(md); + const upb::Handlers* serialize_handlers = serialize_handlercache.Get(md); + const upb::json::ParserMethodPtr parser_method = parse_codecache.Get(md); + ASSERT(serialize_handlers); + + for (const TestCase* test_case = kTestRoundtripMessages; + test_case->input != NULL; test_case++) { + const char *expected = + (test_case->expected == EXPECT_SAME) ? + test_case->input : + test_case->expected; + + for (size_t i = 0; i < strlen(test_case->input); i++) { + test_json_roundtrip_message(test_case->input, expected, + serialize_handlers, parser_method, i); + } + } + + serialize_handlercache = upb::json::PrinterPtr::NewCache(true); + serialize_handlers = serialize_handlercache.Get(md); + + for (const TestCase* test_case = kTestRoundtripMessagesPreserve; + test_case->input != NULL; test_case++) { + const char *expected = + (test_case->expected == EXPECT_SAME) ? + test_case->input : + test_case->expected; + + for (size_t i = 0; i < strlen(test_case->input); i++) { + test_json_roundtrip_message(test_case->input, expected, + serialize_handlers, parser_method, i); + } + } +} + +extern "C" { +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + test_json_roundtrip(); + return 0; +} +} diff --git a/tests/pb/test_decoder.cc b/tests/pb/test_decoder.cc new file mode 100644 index 00000000000..5cc59e40c92 --- /dev/null +++ b/tests/pb/test_decoder.cc @@ -0,0 +1,1203 @@ +/* + * + * An exhaustive set of tests for parsing both valid and invalid protobuf + * input, with buffer breaks in arbitrary places. + * + * Tests to add: + * - string/bytes + * - unknown field handler called appropriately + * - unknown fields can be inserted in random places + * - fuzzing of valid input + * - resource limits (max stack depth, max string len) + * - testing of groups + * - more throrough testing of sequences + * - test skipping of submessages + * - test suspending the decoder + * - buffers that are close enough to the end of the address space that + * pointers overflow (this might be difficult). + * - a few "kitchen sink" examples (one proto that uses all types, lots + * of submsg/sequences, etc. + * - test different handlers at every level and whether handlers fire at + * the correct field path. + * - test skips that extend past the end of current buffer (where decoder + * returns value greater than the size param). + */ + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS // For PRIuS, etc. +#endif + +#include +#include +#include +#include +#include +#include + +#include "tests/test_util.h" +#include "tests/upb_test.h" +#include "tests/pb/test_decoder.upbdefs.h" + +#ifdef AMALGAMATED +#include "upb.h" +#else // AMALGAMATED +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/pb/varint.int.h" +#include "upb/upb.h" +#endif // !AMALGAMATED + +#include "upb/port_def.inc" + +#undef PRINT_FAILURE +#define PRINT_FAILURE(expr) \ + fprintf(stderr, "Assertion failed: %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, "expr: %s\n", #expr); \ + if (testhash) { \ + fprintf(stderr, "assertion failed running test %x.\n", testhash); \ + if (!filter_hash) { \ + fprintf(stderr, \ + "Run with the arg %x to run only this test. " \ + "(This will also turn on extra debugging output)\n", \ + testhash); \ + } \ + fprintf(stderr, "Failed at %02.2f%% through tests.\n", \ + (float)completed * 100 / total); \ + } + +#define MAX_NESTING 64 + +#define LINE(x) x "\n" + +uint32_t filter_hash = 0; +double completed; +double total; +double *count; + +enum TestMode { + COUNT_ONLY = 1, + NO_HANDLERS = 2, + ALL_HANDLERS = 3 +} test_mode; + +// Copied from decoder.c, since this is not a public interface. +typedef struct { + uint8_t native_wire_type; + bool is_numeric; +} upb_decoder_typeinfo; + +static const upb_decoder_typeinfo upb_decoder_types[] = { + {UPB_WIRE_TYPE_END_GROUP, false}, // ENDGROUP + {UPB_WIRE_TYPE_64BIT, true}, // DOUBLE + {UPB_WIRE_TYPE_32BIT, true}, // FLOAT + {UPB_WIRE_TYPE_VARINT, true}, // INT64 + {UPB_WIRE_TYPE_VARINT, true}, // UINT64 + {UPB_WIRE_TYPE_VARINT, true}, // INT32 + {UPB_WIRE_TYPE_64BIT, true}, // FIXED64 + {UPB_WIRE_TYPE_32BIT, true}, // FIXED32 + {UPB_WIRE_TYPE_VARINT, true}, // BOOL + {UPB_WIRE_TYPE_DELIMITED, false}, // STRING + {UPB_WIRE_TYPE_START_GROUP, false}, // GROUP + {UPB_WIRE_TYPE_DELIMITED, false}, // MESSAGE + {UPB_WIRE_TYPE_DELIMITED, false}, // BYTES + {UPB_WIRE_TYPE_VARINT, true}, // UINT32 + {UPB_WIRE_TYPE_VARINT, true}, // ENUM + {UPB_WIRE_TYPE_32BIT, true}, // SFIXED32 + {UPB_WIRE_TYPE_64BIT, true}, // SFIXED64 + {UPB_WIRE_TYPE_VARINT, true}, // SINT32 + {UPB_WIRE_TYPE_VARINT, true}, // SINT64 +}; + +#ifndef USE_GOOGLE +using std::string; +#endif + +void vappendf(string* str, const char *format, va_list args) { + va_list copy; + _upb_va_copy(copy, args); + + int count = vsnprintf(NULL, 0, format, args); + if (count >= 0) + { + UPB_ASSERT(count < 32768); + char *buffer = new char[count + 1]; + UPB_ASSERT(buffer); + count = vsnprintf(buffer, count + 1, format, copy); + UPB_ASSERT(count >= 0); + str->append(buffer, count); + delete [] buffer; + } + va_end(copy); +} + +void appendf(string* str, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + vappendf(str, fmt, args); + va_end(args); +} + +void PrintBinary(const string& str) { + for (size_t i = 0; i < str.size(); i++) { + if (isprint(str[i])) { + fprintf(stderr, "%c", str[i]); + } else { + fprintf(stderr, "\\x%02x", (int)(uint8_t)str[i]); + } + } +} + +/* Routines for building arbitrary protos *************************************/ + +const string empty; + +string cat(const string& a, const string& b, + const string& c = empty, + const string& d = empty, + const string& e = empty, + const string& f = empty, + const string& g = empty, + const string& h = empty, + const string& i = empty, + const string& j = empty, + const string& k = empty, + const string& l = empty) { + string ret; + ret.reserve(a.size() + b.size() + c.size() + d.size() + e.size() + f.size() + + g.size() + h.size() + i.size() + j.size() + k.size() + l.size()); + ret.append(a); + ret.append(b); + ret.append(c); + ret.append(d); + ret.append(e); + ret.append(f); + ret.append(g); + ret.append(h); + ret.append(i); + ret.append(j); + ret.append(k); + ret.append(l); + return ret; +} + +template +string num2string(T num) { + std::ostringstream ss; + ss << num; + return ss.str(); +} + +string varint(uint64_t x) { + char buf[UPB_PB_VARINT_MAX_LEN]; + size_t len = upb_vencode64(x, buf); + return string(buf, len); +} + +// TODO: proper byte-swapping for big-endian machines. +string fixed32(void *data) { return string(static_cast(data), 4); } +string fixed64(void *data) { return string(static_cast(data), 8); } + +string delim(const string& buf) { return cat(varint(buf.size()), buf); } +string uint32(uint32_t u32) { return fixed32(&u32); } +string uint64(uint64_t u64) { return fixed64(&u64); } +string flt(float f) { return fixed32(&f); } +string dbl(double d) { return fixed64(&d); } +string zz32(int32_t x) { return varint(upb_zzenc_32(x)); } +string zz64(int64_t x) { return varint(upb_zzenc_64(x)); } + +string tag(uint32_t fieldnum, char wire_type) { + return varint((fieldnum << 3) | wire_type); +} + +string submsg(uint32_t fn, const string& buf) { + return cat( tag(fn, UPB_WIRE_TYPE_DELIMITED), delim(buf) ); +} + +string group(uint32_t fn, const string& buf) { + return cat(tag(fn, UPB_WIRE_TYPE_START_GROUP), buf, + tag(fn, UPB_WIRE_TYPE_END_GROUP)); +} + +// Like delim()/submsg(), but intentionally encodes an incorrect length. +// These help test when a delimited boundary doesn't land in the right place. +string badlen_delim(int err, const string& buf) { + return cat(varint(buf.size() + err), buf); +} + +string badlen_submsg(int err, uint32_t fn, const string& buf) { + return cat( tag(fn, UPB_WIRE_TYPE_DELIMITED), badlen_delim(err, buf) ); +} + + +/* A set of handlers that covers all .proto types *****************************/ + +// The handlers simply append to a string indicating what handlers were called. +// This string is similar to protobuf text format but fields are referred to by +// number instead of name and sequences are explicitly delimited. We indent +// using the closure depth to test that the stack of closures is properly +// handled. + +int closures[MAX_NESTING]; +string output; + +void indentbuf(string *buf, int depth) { + buf->append(2 * depth, ' '); +} + +#define NUMERIC_VALUE_HANDLER(member, ctype, fmt) \ + bool value_##member(int* depth, const uint32_t* num, ctype val) { \ + indentbuf(&output, *depth); \ + appendf(&output, "%" PRIu32 ":%" fmt "\n", *num, val); \ + return true; \ + } + +NUMERIC_VALUE_HANDLER(uint32, uint32_t, PRIu32) +NUMERIC_VALUE_HANDLER(uint64, uint64_t, PRIu64) +NUMERIC_VALUE_HANDLER(int32, int32_t, PRId32) +NUMERIC_VALUE_HANDLER(int64, int64_t, PRId64) +NUMERIC_VALUE_HANDLER(float, float, "g") +NUMERIC_VALUE_HANDLER(double, double, "g") + +bool value_bool(int* depth, const uint32_t* num, bool val) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":%s\n", *num, val ? "true" : "false"); + return true; +} + +int* startstr(int* depth, const uint32_t* num, size_t size_hint) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":(%zu)\"", *num, size_hint); + return depth + 1; +} + +size_t value_string(int* depth, const uint32_t* num, const char* buf, + size_t n, const upb_bufhandle* handle) { + UPB_UNUSED(num); + UPB_UNUSED(depth); + output.append(buf, n); + ASSERT(handle == &global_handle); + return n; +} + +bool endstr(int* depth, const uint32_t* num) { + UPB_UNUSED(num); + output.append("\n"); + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":\"\n", *num); + return true; +} + +int* startsubmsg(int* depth, const uint32_t* num) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":{\n", *num); + return depth + 1; +} + +bool endsubmsg(int* depth, const uint32_t* num) { + UPB_UNUSED(num); + indentbuf(&output, *depth); + output.append("}\n"); + return true; +} + +int* startseq(int* depth, const uint32_t* num) { + indentbuf(&output, *depth); + appendf(&output, "%" PRIu32 ":[\n", *num); + return depth + 1; +} + +bool endseq(int* depth, const uint32_t* num) { + UPB_UNUSED(num); + indentbuf(&output, *depth); + output.append("]\n"); + return true; +} + +bool startmsg(int* depth) { + indentbuf(&output, *depth); + output.append("<\n"); + return true; +} + +bool endmsg(int* depth, upb_status* status) { + UPB_UNUSED(status); + indentbuf(&output, *depth); + output.append(">\n"); + return true; +} + +void free_uint32(void *val) { + uint32_t *u32 = static_cast(val); + delete u32; +} + +template +void doreg(upb::HandlersPtr h, uint32_t num) { + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(num); + ASSERT(f); + ASSERT(h.SetValueHandler(f, UpbBind(F, new uint32_t(num)))); + if (f.IsSequence()) { + ASSERT(h.SetStartSequenceHandler(f, UpbBind(startseq, new uint32_t(num)))); + ASSERT(h.SetEndSequenceHandler(f, UpbBind(endseq, new uint32_t(num)))); + } +} + +// The repeated field number to correspond to the given non-repeated field +// number. +uint32_t rep_fn(uint32_t fn) { + return (UPB_MAX_FIELDNUMBER - 1000) + fn; +} + +#define NOP_FIELD 40 +#define UNKNOWN_FIELD 666 + +template +void reg(upb::HandlersPtr h, upb_descriptortype_t type) { + // We register both a repeated and a non-repeated field for every type. + // For the non-repeated field we make the field number the same as the + // type. For the repeated field we make it a function of the type. + doreg(h, type); + doreg(h, rep_fn(type)); +} + +void regseq(upb::HandlersPtr h, upb::FieldDefPtr f, uint32_t num) { + ASSERT(h.SetStartSequenceHandler(f, UpbBind(startseq, new uint32_t(num)))); + ASSERT(h.SetEndSequenceHandler(f, UpbBind(endseq, new uint32_t(num)))); +} + +void reg_subm(upb::HandlersPtr h, uint32_t num) { + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(num); + ASSERT(f); + if (f.IsSequence()) regseq(h, f, num); + ASSERT( + h.SetStartSubMessageHandler(f, UpbBind(startsubmsg, new uint32_t(num)))); + ASSERT(h.SetEndSubMessageHandler(f, UpbBind(endsubmsg, new uint32_t(num)))); +} + +void reg_str(upb::HandlersPtr h, uint32_t num) { + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(num); + ASSERT(f); + if (f.IsSequence()) regseq(h, f, num); + ASSERT(h.SetStartStringHandler(f, UpbBind(startstr, new uint32_t(num)))); + ASSERT(h.SetEndStringHandler(f, UpbBind(endstr, new uint32_t(num)))); + ASSERT(h.SetStringHandler(f, UpbBind(value_string, new uint32_t(num)))); +} + +struct HandlerRegisterData { + TestMode mode; +}; + +void callback(const void *closure, upb::Handlers* h_ptr) { + upb::HandlersPtr h(h_ptr); + const HandlerRegisterData* data = + static_cast(closure); + if (data->mode == ALL_HANDLERS) { + h.SetStartMessageHandler(UpbMakeHandler(startmsg)); + h.SetEndMessageHandler(UpbMakeHandler(endmsg)); + + // Register handlers for each type. + reg(h, UPB_DESCRIPTOR_TYPE_DOUBLE); + reg (h, UPB_DESCRIPTOR_TYPE_FLOAT); + reg (h, UPB_DESCRIPTOR_TYPE_INT64); + reg(h, UPB_DESCRIPTOR_TYPE_UINT64); + reg (h, UPB_DESCRIPTOR_TYPE_INT32); + reg(h, UPB_DESCRIPTOR_TYPE_FIXED64); + reg(h, UPB_DESCRIPTOR_TYPE_FIXED32); + reg (h, UPB_DESCRIPTOR_TYPE_BOOL); + reg(h, UPB_DESCRIPTOR_TYPE_UINT32); + reg (h, UPB_DESCRIPTOR_TYPE_ENUM); + reg (h, UPB_DESCRIPTOR_TYPE_SFIXED32); + reg (h, UPB_DESCRIPTOR_TYPE_SFIXED64); + reg (h, UPB_DESCRIPTOR_TYPE_SINT32); + reg (h, UPB_DESCRIPTOR_TYPE_SINT64); + + reg_str(h, UPB_DESCRIPTOR_TYPE_STRING); + reg_str(h, UPB_DESCRIPTOR_TYPE_BYTES); + reg_str(h, rep_fn(UPB_DESCRIPTOR_TYPE_STRING)); + reg_str(h, rep_fn(UPB_DESCRIPTOR_TYPE_BYTES)); + + // Register submessage/group handlers that are self-recursive + // to this type, eg: message M { optional M m = 1; } + reg_subm(h, UPB_DESCRIPTOR_TYPE_MESSAGE); + reg_subm(h, rep_fn(UPB_DESCRIPTOR_TYPE_MESSAGE)); + + if (h.message_def().full_name() == std::string("DecoderTest")) { + reg_subm(h, UPB_DESCRIPTOR_TYPE_GROUP); + reg_subm(h, rep_fn(UPB_DESCRIPTOR_TYPE_GROUP)); + } + + // For NOP_FIELD we register no handlers, so we can pad a proto freely without + // changing the output. + } +} + +/* Running of test cases ******************************************************/ + +const upb::Handlers *global_handlers; +upb::pb::DecoderMethodPtr global_method; + +upb::pb::DecoderPtr CreateDecoder(upb::Arena* arena, + upb::pb::DecoderMethodPtr method, + upb::Sink sink, upb::Status* status) { + upb::pb::DecoderPtr ret = + upb::pb::DecoderPtr::Create(arena, method, sink, status); + ret.set_max_nesting(MAX_NESTING); + return ret; +} + +uint32_t Hash(const string& proto, const string* expected_output, size_t seam1, + size_t seam2, bool may_skip) { + uint32_t hash = upb_murmur_hash2(proto.c_str(), proto.size(), 0); + if (expected_output) + hash = upb_murmur_hash2(expected_output->c_str(), expected_output->size(), hash); + hash = upb_murmur_hash2(&seam1, sizeof(seam1), hash); + hash = upb_murmur_hash2(&seam2, sizeof(seam2), hash); + hash = upb_murmur_hash2(&may_skip, sizeof(may_skip), hash); + return hash; +} + +void CheckBytesParsed(upb::pb::DecoderPtr decoder, size_t ofs) { + // We can't have parsed more data than the decoder callback is telling us it + // parsed. + ASSERT(decoder.BytesParsed() <= ofs); + + // The difference between what we've decoded and what the decoder has accepted + // represents the internally buffered amount. This amount should not exceed + // this value which comes from decoder.int.h. + ASSERT(ofs <= (decoder.BytesParsed() + UPB_DECODER_MAX_RESIDUAL_BYTES)); +} + +static bool parse(VerboseParserEnvironment* env, + upb::pb::DecoderPtr decoder, int bytes) { + CheckBytesParsed(decoder, env->ofs()); + bool ret = env->ParseBuffer(bytes); + if (ret) { + CheckBytesParsed(decoder, env->ofs()); + } + + return ret; +} + +void do_run_decoder(VerboseParserEnvironment* env, upb::pb::DecoderPtr decoder, + const string& proto, const string* expected_output, + size_t i, size_t j, bool may_skip) { + env->Reset(proto.c_str(), proto.size(), may_skip, expected_output == NULL); + decoder.Reset(); + + testhash = Hash(proto, expected_output, i, j, may_skip); + if (filter_hash && testhash != filter_hash) return; + if (test_mode != COUNT_ONLY) { + output.clear(); + + if (filter_hash) { + fprintf(stderr, "RUNNING TEST CASE, hash=%x\n", testhash); + fprintf(stderr, "Input (len=%u): ", (unsigned)proto.size()); + PrintBinary(proto); + fprintf(stderr, "\n"); + if (expected_output) { + if (test_mode == ALL_HANDLERS) { + fprintf(stderr, "Expected output: %s\n", expected_output->c_str()); + } else if (test_mode == NO_HANDLERS) { + fprintf(stderr, + "No handlers are registered, BUT if they were " + "the expected output would be: %s\n", + expected_output->c_str()); + } + } else { + fprintf(stderr, "Expected to FAIL\n"); + } + } + + bool ok = env->Start() && + parse(env, decoder, i) && + parse(env, decoder, j - i) && + parse(env, decoder, -1) && + env->End(); + + ASSERT(env->CheckConsistency()); + + if (test_mode == ALL_HANDLERS) { + if (expected_output) { + if (output != *expected_output) { + fprintf(stderr, "Text mismatch: '%s' vs '%s'\n", + output.c_str(), expected_output->c_str()); + } + ASSERT(ok); + ASSERT(output == *expected_output); + } else { + if (ok) { + fprintf(stderr, "Didn't expect ok result, but got output: '%s'\n", + output.c_str()); + } + ASSERT(!ok); + } + } + } + (*count)++; +} + +void run_decoder(const string& proto, const string* expected_output) { + VerboseParserEnvironment env(filter_hash != 0); + upb::Sink sink(global_handlers, &closures[0]); + upb::pb::DecoderPtr decoder = CreateDecoder(env.arena(), global_method, sink, env.status()); + env.ResetBytesSink(decoder.input()); + for (size_t i = 0; i < proto.size(); i++) { + for (size_t j = i; j < UPB_MIN(proto.size(), i + 5); j++) { + do_run_decoder(&env, decoder, proto, expected_output, i, j, true); + if (env.SkippedWithNull()) { + do_run_decoder(&env, decoder, proto, expected_output, i, j, false); + } + } + } + testhash = 0; +} + +const static string thirty_byte_nop = cat( + tag(NOP_FIELD, UPB_WIRE_TYPE_DELIMITED), delim(string(30, 'X')) ); + +// Indents and wraps text as if it were a submessage with this field number +string wrap_text(int32_t fn, const string& text) { + string wrapped_text = text; + size_t pos = 0; + string replace_with = "\n "; + while ((pos = wrapped_text.find("\n", pos)) != string::npos && + pos != wrapped_text.size() - 1) { + wrapped_text.replace(pos, 1, replace_with); + pos += replace_with.size(); + } + wrapped_text = cat( + LINE("<"), + num2string(fn), LINE(":{") + " ", wrapped_text, + LINE("}") + LINE(">")); + return wrapped_text; +} + +void assert_successful_parse(const string& proto, + const char *expected_fmt, ...) { + string expected_text; + va_list args; + va_start(args, expected_fmt); + vappendf(&expected_text, expected_fmt, args); + va_end(args); + // To test both middle-of-buffer and end-of-buffer code paths, + // repeat once with no-op padding data at the end of buffer. + run_decoder(proto, &expected_text); + run_decoder(cat( proto, thirty_byte_nop ), &expected_text); + + // Test that this also works when wrapped in a submessage or group. + // Indent the expected text one level and wrap it. + string wrapped_text1 = wrap_text(UPB_DESCRIPTOR_TYPE_MESSAGE, expected_text); + string wrapped_text2 = wrap_text(UPB_DESCRIPTOR_TYPE_GROUP, expected_text); + + run_decoder(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, proto), &wrapped_text1); + run_decoder(group(UPB_DESCRIPTOR_TYPE_GROUP, proto), &wrapped_text2); +} + +void assert_does_not_parse_at_eof(const string& proto) { + run_decoder(proto, NULL); + + // Also test that we fail to parse at end-of-submessage, not just + // end-of-message. But skip this if we have no handlers, because in that + // case we won't descend into the submessage. + if (test_mode != NO_HANDLERS) { + run_decoder(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, proto), NULL); + run_decoder(cat(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, proto), + thirty_byte_nop), NULL); + } +} + +void assert_does_not_parse(const string& proto) { + // Test that the error is caught both at end-of-buffer and middle-of-buffer. + assert_does_not_parse_at_eof(proto); + assert_does_not_parse_at_eof(cat( proto, thirty_byte_nop )); +} + + +/* The actual tests ***********************************************************/ + +void test_premature_eof_for_type(upb_descriptortype_t type) { + // Incomplete values for each wire type. + static const string incompletes[6] = { + string("\x80"), // UPB_WIRE_TYPE_VARINT + string("abcdefg"), // UPB_WIRE_TYPE_64BIT + string("\x80"), // UPB_WIRE_TYPE_DELIMITED (partial length) + string(), // UPB_WIRE_TYPE_START_GROUP (no value required) + string(), // UPB_WIRE_TYPE_END_GROUP (no value required) + string("abc") // UPB_WIRE_TYPE_32BIT + }; + + uint32_t fieldnum = type; + uint32_t rep_fieldnum = rep_fn(type); + int wire_type = upb_decoder_types[type].native_wire_type; + const string& incomplete = incompletes[wire_type]; + + // EOF before a known non-repeated value. + assert_does_not_parse_at_eof(tag(fieldnum, wire_type)); + + // EOF before a known repeated value. + assert_does_not_parse_at_eof(tag(rep_fieldnum, wire_type)); + + // EOF before an unknown value. + assert_does_not_parse_at_eof(tag(UNKNOWN_FIELD, wire_type)); + + // EOF inside a known non-repeated value. + assert_does_not_parse_at_eof( + cat( tag(fieldnum, wire_type), incomplete )); + + // EOF inside a known repeated value. + assert_does_not_parse_at_eof( + cat( tag(rep_fieldnum, wire_type), incomplete )); + + // EOF inside an unknown value. + assert_does_not_parse_at_eof( + cat( tag(UNKNOWN_FIELD, wire_type), incomplete )); + + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + // EOF in the middle of delimited data for known non-repeated value. + assert_does_not_parse_at_eof( + cat( tag(fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for known repeated value. + assert_does_not_parse_at_eof( + cat( tag(rep_fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for unknown value. + assert_does_not_parse_at_eof( + cat( tag(UNKNOWN_FIELD, wire_type), varint(1) )); + + if (type == UPB_DESCRIPTOR_TYPE_MESSAGE) { + // Submessage ends in the middle of a value. + string incomplete_submsg = + cat ( tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), + incompletes[UPB_WIRE_TYPE_VARINT] ); + assert_does_not_parse( + cat( tag(fieldnum, UPB_WIRE_TYPE_DELIMITED), + varint(incomplete_submsg.size()), + incomplete_submsg )); + } + } else { + // Packed region ends in the middle of a value. + assert_does_not_parse( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), + varint(incomplete.size()), + incomplete )); + + // EOF in the middle of packed region. + assert_does_not_parse_at_eof( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), varint(1) )); + } +} + +// "33" and "66" are just two random values that all numeric types can +// represent. +void test_valid_data_for_type(upb_descriptortype_t type, + const string& enc33, const string& enc66) { + uint32_t fieldnum = type; + uint32_t rep_fieldnum = rep_fn(type); + int wire_type = upb_decoder_types[type].native_wire_type; + + // Non-repeated + assert_successful_parse( + cat( tag(fieldnum, wire_type), enc33, + tag(fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:33") + LINE("%u:66") + LINE(">"), fieldnum, fieldnum); + + // Non-packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, wire_type), enc33, + tag(rep_fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); + + // Packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), + delim(cat( enc33, enc66 )) ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); +} + +void test_valid_data_for_signed_type(upb_descriptortype_t type, + const string& enc33, const string& enc66) { + uint32_t fieldnum = type; + uint32_t rep_fieldnum = rep_fn(type); + int wire_type = upb_decoder_types[type].native_wire_type; + + // Non-repeated + assert_successful_parse( + cat( tag(fieldnum, wire_type), enc33, + tag(fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:33") + LINE("%u:-66") + LINE(">"), fieldnum, fieldnum); + + // Non-packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, wire_type), enc33, + tag(rep_fieldnum, wire_type), enc66 ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:-66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); + + // Packed repeated. + assert_successful_parse( + cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED), + delim(cat( enc33, enc66 )) ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE(" %u:-66") + LINE("]") + LINE(">"), rep_fieldnum, rep_fieldnum, rep_fieldnum); +} + +// Test that invalid protobufs are properly detected (without crashing) and +// have an error reported. Field numbers match registered handlers above. +void test_invalid() { + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_DOUBLE); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_FLOAT); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_INT64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_UINT64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_INT32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_FIXED64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_FIXED32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_BOOL); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_STRING); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_BYTES); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_UINT32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_ENUM); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SFIXED32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SFIXED64); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SINT32); + test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SINT64); + + // EOF inside a tag's varint. + assert_does_not_parse_at_eof( string("\x80") ); + + // EOF inside a known group. + // TODO(haberman): add group to decoder test schema. + //assert_does_not_parse_at_eof( tag(4, UPB_WIRE_TYPE_START_GROUP) ); + + // EOF inside an unknown group. + assert_does_not_parse_at_eof( tag(UNKNOWN_FIELD, UPB_WIRE_TYPE_START_GROUP) ); + + // End group that we are not currently in. + assert_does_not_parse( tag(4, UPB_WIRE_TYPE_END_GROUP) ); + + // Field number is 0. + assert_does_not_parse( + cat( tag(0, UPB_WIRE_TYPE_DELIMITED), varint(0) )); + // The previous test alone did not catch this particular pattern which could + // corrupt the internal state. + assert_does_not_parse( + cat( tag(0, UPB_WIRE_TYPE_64BIT), uint64(0) )); + + // Field number is too large. + assert_does_not_parse( + cat( tag(UPB_MAX_FIELDNUMBER + 1, UPB_WIRE_TYPE_DELIMITED), + varint(0) )); + + // Known group inside a submessage has ENDGROUP tag AFTER submessage end. + assert_does_not_parse( + cat ( submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, + tag(UPB_DESCRIPTOR_TYPE_GROUP, UPB_WIRE_TYPE_START_GROUP)), + tag(UPB_DESCRIPTOR_TYPE_GROUP, UPB_WIRE_TYPE_END_GROUP))); + + // Unknown string extends past enclosing submessage. + assert_does_not_parse( + cat (badlen_submsg(-1, UPB_DESCRIPTOR_TYPE_MESSAGE, + submsg(12345, string(" "))), + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, string(" ")))); + + // Unknown fixed-length field extends past enclosing submessage. + assert_does_not_parse( + cat (badlen_submsg(-1, UPB_DESCRIPTOR_TYPE_MESSAGE, + cat( tag(12345, UPB_WIRE_TYPE_64BIT), uint64(0))), + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, string(" ")))); + + // Test exceeding the resource limit of stack depth. + if (test_mode != NO_HANDLERS) { + string buf; + for (int i = 0; i <= MAX_NESTING; i++) { + buf.assign(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, buf)); + } + assert_does_not_parse(buf); + } +} + +void test_valid() { + // Empty protobuf. + assert_successful_parse(string(""), "<\n>\n"); + + // Empty protobuf where we never call PutString between + // StartString/EndString. + + // Randomly generated hash for this test, hope it doesn't conflict with others + // by chance. + const uint32_t emptyhash = 0x5709be8e; + if (!filter_hash || filter_hash == testhash) { + testhash = emptyhash; + upb::Status status; + upb::Arena arena; + upb::Sink sink(global_handlers, &closures[0]); + upb::pb::DecoderPtr decoder = + CreateDecoder(&arena, global_method, sink, &status); + output.clear(); + bool ok = upb::PutBuffer(std::string(), decoder.input()); + ASSERT(ok); + ASSERT(status.ok()); + if (test_mode == ALL_HANDLERS) { + ASSERT(output == string("<\n>\n")); + } + } + + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_DOUBLE, + dbl(33), + dbl(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_FLOAT, flt(33), flt(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_INT64, + varint(33), + varint(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_INT32, + varint(33), + varint(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_ENUM, + varint(33), + varint(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SFIXED32, + uint32(33), + uint32(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SFIXED64, + uint64(33), + uint64(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SINT32, + zz32(33), + zz32(-66)); + test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_SINT64, + zz64(33), + zz64(-66)); + + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_UINT64, varint(33), varint(66)); + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_UINT32, varint(33), varint(66)); + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_FIXED64, uint64(33), uint64(66)); + test_valid_data_for_type(UPB_DESCRIPTOR_TYPE_FIXED32, uint32(33), uint32(66)); + + // Unknown fields. + int int32_type = UPB_DESCRIPTOR_TYPE_INT32; + int msg_type = UPB_DESCRIPTOR_TYPE_MESSAGE; + assert_successful_parse( + cat( tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678) ), + "<\n>\n"); + assert_successful_parse( + cat( tag(12345, UPB_WIRE_TYPE_32BIT), uint32(2345678) ), + "<\n>\n"); + assert_successful_parse( + cat( tag(12345, UPB_WIRE_TYPE_64BIT), uint64(2345678) ), + "<\n>\n"); + assert_successful_parse( + submsg(12345, string(" ")), + "<\n>\n"); + + // Unknown field inside a known submessage. + assert_successful_parse( + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, submsg(12345, string(" "))), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" >") + LINE("}") + LINE(">"), UPB_DESCRIPTOR_TYPE_MESSAGE); + + assert_successful_parse( + cat (submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, submsg(12345, string(" "))), + tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), + varint(5)), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" >") + LINE("}") + LINE("%u:5") + LINE(">"), UPB_DESCRIPTOR_TYPE_MESSAGE, UPB_DESCRIPTOR_TYPE_INT32); + + // This triggered a previous bug in the decoder. + assert_successful_parse( + cat( tag(UPB_DESCRIPTOR_TYPE_SFIXED32, UPB_WIRE_TYPE_VARINT), + varint(0) ), + "<\n>\n"); + + assert_successful_parse( + cat( + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, + submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, + cat( tag(int32_type, UPB_WIRE_TYPE_VARINT), varint(2345678), + tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678) ))), + tag(int32_type, UPB_WIRE_TYPE_VARINT), varint(22222)), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" %u:{") + LINE(" <") + LINE(" %u:2345678") + LINE(" >") + LINE(" }") + LINE(" >") + LINE("}") + LINE("%u:22222") + LINE(">"), msg_type, msg_type, int32_type, int32_type); + + assert_successful_parse( + cat( tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), varint(1), + tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678) ), + LINE("<") + LINE("%u:1") + LINE(">"), UPB_DESCRIPTOR_TYPE_INT32); + + // String inside submsg. + uint32_t msg_fn = UPB_DESCRIPTOR_TYPE_MESSAGE; + assert_successful_parse( + submsg(msg_fn, + cat ( tag(UPB_DESCRIPTOR_TYPE_STRING, UPB_WIRE_TYPE_DELIMITED), + delim(string("abcde")) + ) + ), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" %u:(5)\"abcde") + LINE(" %u:\"") + LINE(" >") + LINE("}") + LINE(">"), msg_fn, UPB_DESCRIPTOR_TYPE_STRING, + UPB_DESCRIPTOR_TYPE_STRING); + + // Test implicit startseq/endseq. + uint32_t repfl_fn = rep_fn(UPB_DESCRIPTOR_TYPE_FLOAT); + uint32_t repdb_fn = rep_fn(UPB_DESCRIPTOR_TYPE_DOUBLE); + assert_successful_parse( + cat( tag(repfl_fn, UPB_WIRE_TYPE_32BIT), flt(33), + tag(repdb_fn, UPB_WIRE_TYPE_64BIT), dbl(66) ), + LINE("<") + LINE("%u:[") + LINE(" %u:33") + LINE("]") + LINE("%u:[") + LINE(" %u:66") + LINE("]") + LINE(">"), repfl_fn, repfl_fn, repdb_fn, repdb_fn); + + // Submessage tests. + assert_successful_parse( + submsg(msg_fn, submsg(msg_fn, submsg(msg_fn, string()))), + LINE("<") + LINE("%u:{") + LINE(" <") + LINE(" %u:{") + LINE(" <") + LINE(" %u:{") + LINE(" <") + LINE(" >") + LINE(" }") + LINE(" >") + LINE(" }") + LINE(" >") + LINE("}") + LINE(">"), msg_fn, msg_fn, msg_fn); + + uint32_t repm_fn = rep_fn(UPB_DESCRIPTOR_TYPE_MESSAGE); + assert_successful_parse( + submsg(repm_fn, submsg(repm_fn, string())), + LINE("<") + LINE("%u:[") + LINE(" %u:{") + LINE(" <") + LINE(" %u:[") + LINE(" %u:{") + LINE(" <") + LINE(" >") + LINE(" }") + LINE(" ]") + LINE(" >") + LINE(" }") + LINE("]") + LINE(">"), repm_fn, repm_fn, repm_fn, repm_fn); + + // Test unknown group. + uint32_t unknown_group_fn = 12321; + assert_successful_parse( + cat( tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) ), + LINE("<") + LINE(">") + ); + + // Test some unknown fields inside an unknown group. + const string unknown_group_with_data = + cat( + tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678), + tag(123456789, UPB_WIRE_TYPE_32BIT), uint32(2345678), + tag(123477, UPB_WIRE_TYPE_64BIT), uint64(2345678), + tag(123, UPB_WIRE_TYPE_DELIMITED), varint(0), + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) + ); + + // Nested unknown group with data. + assert_successful_parse( + cat( + tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + unknown_group_with_data, + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP), + tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), varint(1) + ), + LINE("<") + LINE("%u:1") + LINE(">"), + UPB_DESCRIPTOR_TYPE_INT32 + ); + + assert_successful_parse( + cat( tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP), + tag(unknown_group_fn + 1, UPB_WIRE_TYPE_START_GROUP), + tag(unknown_group_fn + 1, UPB_WIRE_TYPE_END_GROUP), + tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) ), + LINE("<") + LINE(">") + ); + + // Staying within the stack limit should work properly. + string buf; + string textbuf; + int total = MAX_NESTING - 1; + for (int i = 0; i < total; i++) { + buf.assign(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, buf)); + indentbuf(&textbuf, i); + textbuf.append("<\n"); + indentbuf(&textbuf, i); + appendf(&textbuf, "%u:{\n", UPB_DESCRIPTOR_TYPE_MESSAGE); + } + indentbuf(&textbuf, total); + textbuf.append("<\n"); + indentbuf(&textbuf, total); + textbuf.append(">\n"); + for (int i = 0; i < total; i++) { + indentbuf(&textbuf, total - i - 1); + textbuf.append("}\n"); + indentbuf(&textbuf, total - i - 1); + textbuf.append(">\n"); + } + // Have to use run_decoder directly, because we are at max nesting and can't + // afford the extra nesting that assert_successful_parse() will do. + run_decoder(buf, &textbuf); +} + +void empty_callback(const void *closure, upb::Handlers* h_ptr) {} + +void test_emptyhandlers(upb::SymbolTable* symtab) { + // Create an empty handlers to make sure that the decoder can handle empty + // messages. + HandlerRegisterData handlerdata; + handlerdata.mode = test_mode; + + upb::HandlerCache handler_cache(empty_callback, &handlerdata); + upb::pb::CodeCache pb_code_cache(&handler_cache); + + upb::MessageDefPtr md = upb::MessageDefPtr(Empty_getmsgdef(symtab->ptr())); + global_handlers = handler_cache.Get(md); + global_method = pb_code_cache.Get(md); + + // TODO: also test the case where a message has fields, but the fields are + // submessage fields and have no handlers. This also results in a decoder + // method with no field-handling code. + + // Ensure that the method can run with empty and non-empty input. + string test_unknown_field_msg = + cat(tag(1, UPB_WIRE_TYPE_VARINT), varint(42), + tag(2, UPB_WIRE_TYPE_DELIMITED), delim("My test data")); + const struct { + const char* data; + size_t length; + } testdata[] = { + { "", 0 }, + { test_unknown_field_msg.data(), test_unknown_field_msg.size() }, + { NULL, 0 }, + }; + for (int i = 0; testdata[i].data; i++) { + VerboseParserEnvironment env(filter_hash != 0); + upb::Sink sink(global_method.dest_handlers(), &closures[0]); + upb::pb::DecoderPtr decoder = + CreateDecoder(env.arena(), global_method, sink, env.status()); + env.ResetBytesSink(decoder.input()); + env.Reset(testdata[i].data, testdata[i].length, true, false); + ASSERT(env.Start()); + ASSERT(env.ParseBuffer(-1)); + ASSERT(env.End()); + ASSERT(env.CheckConsistency()); + } +} + +void run_tests() { + HandlerRegisterData handlerdata; + handlerdata.mode = test_mode; + + upb::SymbolTable symtab; + upb::HandlerCache handler_cache(callback, &handlerdata); + upb::pb::CodeCache pb_code_cache(&handler_cache); + + upb::MessageDefPtr md(DecoderTest_getmsgdef(symtab.ptr())); + global_handlers = handler_cache.Get(md); + global_method = pb_code_cache.Get(md); + completed = 0; + + test_invalid(); + test_valid(); + + test_emptyhandlers(&symtab); +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + if (argc > 1) + filter_hash = strtol(argv[1], NULL, 16); + for (int i = 0; i < MAX_NESTING; i++) { + closures[i] = i; + } + + // Count tests. + count = &total; + total = 0; + test_mode = COUNT_ONLY; + run_tests(); + count = &completed; + + total *= 2; // NO_HANDLERS, ALL_HANDLERS. + + test_mode = NO_HANDLERS; + run_tests(); + + test_mode = ALL_HANDLERS; + run_tests(); + + printf("All tests passed, %d assertions.\n", num_assertions); + return 0; +} + +} diff --git a/tests/pb/test_decoder.proto b/tests/pb/test_decoder.proto new file mode 100644 index 00000000000..e9fa6ad32ce --- /dev/null +++ b/tests/pb/test_decoder.proto @@ -0,0 +1,128 @@ + +syntax = "proto2"; + +enum TestEnum { + FOO = 1; +} + +message Empty {} + +message DecoderTest { + optional double f_double = 1; + optional float f_float = 2; + optional int64 f_int64 = 3; + optional uint64 f_uint64 = 4; + optional int32 f_int32 = 5; + optional fixed64 f_fixed64 = 6; + optional fixed32 f_fixed32 = 7; + optional bool f_bool = 8; + optional string f_string = 9; + optional DecoderTest f_message = 11; + optional bytes f_bytes = 12; + optional uint32 f_uint32 = 13; + optional TestEnum f_enum = 14; + optional sfixed32 f_sfixed32 = 15; + optional sfixed64 f_sfixed64 = 16; + optional sint32 f_sint32 = 17; + optional sint64 f_sint64 = 18; + + optional string nop_field = 40; + + repeated double r_double = 536869912; + repeated float r_float = 536869913; + repeated int64 r_int64 = 536869914; + repeated uint64 r_uint64 = 536869915; + repeated int32 r_int32 = 536869916; + repeated fixed64 r_fixed64 = 536869917; + repeated fixed32 r_fixed32 = 536869918; + repeated bool r_bool = 536869919; + repeated string r_string = 536869920; + repeated DecoderTest r_message = 536869922; + repeated bytes r_bytes = 536869923; + repeated uint32 r_uint32 = 536869924; + repeated TestEnum r_enum = 536869925; + repeated sfixed32 r_sfixed32 = 536869926; + repeated sfixed64 r_sfixed64 = 536869927; + repeated sint32 r_sint32 = 536869928; + repeated sint64 r_sint64 = 536869929; + + optional group F_group = 10 { + optional double f_double = 1; + optional float f_float = 2; + optional int64 f_int64 = 3; + optional uint64 f_uint64 = 4; + optional int32 f_int32 = 5; + optional fixed64 f_fixed64 = 6; + optional fixed32 f_fixed32 = 7; + optional bool f_bool = 8; + optional string f_string = 9; + optional DecoderTest f_message = 11; + optional bytes f_bytes = 12; + optional uint32 f_uint32 = 13; + optional TestEnum f_enum = 14; + optional sfixed32 f_sfixed32 = 15; + optional sfixed64 f_sfixed64 = 16; + optional sint32 f_sint32 = 17; + optional sint64 f_sint64 = 18; + + optional string nop_field = 40; + + repeated double r_double = 536869912; + repeated float r_float = 536869913; + repeated int64 r_int64 = 536869914; + repeated uint64 r_uint64 = 536869915; + repeated int32 r_int32 = 536869916; + repeated fixed64 r_fixed64 = 536869917; + repeated fixed32 r_fixed32 = 536869918; + repeated bool r_bool = 536869919; + repeated string r_string = 536869920; + repeated DecoderTest r_message = 536869922; + repeated bytes r_bytes = 536869923; + repeated uint32 r_uint32 = 536869924; + repeated TestEnum r_enum = 536869925; + repeated sfixed32 r_sfixed32 = 536869926; + repeated sfixed64 r_sfixed64 = 536869927; + repeated sint32 r_sint32 = 536869928; + repeated sint64 r_sint64 = 536869929; + } + + optional group R_group = 536869921 { + optional double f_double = 1; + optional float f_float = 2; + optional int64 f_int64 = 3; + optional uint64 f_uint64 = 4; + optional int32 f_int32 = 5; + optional fixed64 f_fixed64 = 6; + optional fixed32 f_fixed32 = 7; + optional bool f_bool = 8; + optional string f_string = 9; + optional DecoderTest f_message = 11; + optional bytes f_bytes = 12; + optional uint32 f_uint32 = 13; + optional TestEnum f_enum = 14; + optional sfixed32 f_sfixed32 = 15; + optional sfixed64 f_sfixed64 = 16; + optional sint32 f_sint32 = 17; + optional sint64 f_sint64 = 18; + + optional string nop_field = 40; + + repeated double r_double = 536869912; + repeated float r_float = 536869913; + repeated int64 r_int64 = 536869914; + repeated uint64 r_uint64 = 536869915; + repeated int32 r_int32 = 536869916; + repeated fixed64 r_fixed64 = 536869917; + repeated fixed32 r_fixed32 = 536869918; + repeated bool r_bool = 536869919; + repeated string r_string = 536869920; + repeated DecoderTest r_message = 536869922; + repeated bytes r_bytes = 536869923; + repeated uint32 r_uint32 = 536869924; + repeated TestEnum r_enum = 536869925; + repeated sfixed32 r_sfixed32 = 536869926; + repeated sfixed64 r_sfixed64 = 536869927; + repeated sint32 r_sint32 = 536869928; + repeated sint64 r_sint64 = 536869929; + } +} diff --git a/tests/pb/test_encoder.cc b/tests/pb/test_encoder.cc new file mode 100644 index 00000000000..c6544beea57 --- /dev/null +++ b/tests/pb/test_encoder.cc @@ -0,0 +1,48 @@ + +#include "tests/test_util.h" +#include "tests/upb_test.h" +#include "upb/bindings/stdc++/string.h" +#include "google/protobuf/descriptor.upb.h" +#include "google/protobuf/descriptor.upbdefs.h" +#include "upb/pb/decoder.h" +#include "upb/pb/encoder.h" + +#include "upb/port_def.inc" +#include + +void test_pb_roundtrip() { + std::string input( + google_protobuf_descriptor_proto_upbdefinit.descriptor.data, + google_protobuf_descriptor_proto_upbdefinit.descriptor.size); + std::cout << input.size() << "\n"; + upb::SymbolTable symtab; + upb::HandlerCache encoder_cache(upb::pb::EncoderPtr::NewCache()); + upb::pb::CodeCache decoder_cache(&encoder_cache); + upb::Arena arena; + upb::Status status; + upb::MessageDefPtr md( + google_protobuf_FileDescriptorProto_getmsgdef(symtab.ptr())); + ASSERT(md); + const upb::Handlers *encoder_handlers = encoder_cache.Get(md); + ASSERT(encoder_handlers); + const upb::pb::DecoderMethodPtr method = decoder_cache.Get(md); + + std::string output; + upb::StringSink string_sink(&output); + upb::pb::EncoderPtr encoder = + upb::pb::EncoderPtr::Create(&arena, encoder_handlers, string_sink.input()); + upb::pb::DecoderPtr decoder = + upb::pb::DecoderPtr::Create(&arena, method, encoder.input(), &status); + bool ok = upb::PutBuffer(input, decoder.input()); + ASSERT(ok); + ASSERT(input == output); +} + +extern "C" { +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + test_pb_roundtrip(); + return 0; +} +} diff --git a/tests/pb/test_varint.c b/tests/pb/test_varint.c new file mode 100644 index 00000000000..95a04ab9b8d --- /dev/null +++ b/tests/pb/test_varint.c @@ -0,0 +1,117 @@ + +#include +#include "upb/pb/varint.int.h" +#include "tests/upb_test.h" + +#include "upb/port_def.inc" + +/* Test that we can round-trip from int->varint->int. */ +static void test_varint_for_num(upb_decoderet (*decoder)(const char*), + uint64_t num) { + char buf[16]; + size_t bytes; + upb_decoderet r; + + memset(buf, 0xff, sizeof(buf)); + bytes = upb_vencode64(num, buf); + + if (num <= UINT32_MAX) { + uint64_t encoded = upb_vencode32(num); + char buf2[16]; + upb_decoderet r; + + memset(buf2, 0, sizeof(buf2)); + memcpy(&buf2, &encoded, 8); +#ifdef UPB_BIG_ENDIAN + char swap[8]; + swap[0] = buf2[7]; + swap[1] = buf2[6]; + swap[2] = buf2[5]; + swap[3] = buf2[4]; + swap[4] = buf2[3]; + swap[5] = buf2[2]; + swap[6] = buf2[1]; + swap[7] = buf2[0]; + buf2[0] = swap[0]; + buf2[1] = swap[1]; + buf2[2] = swap[2]; + buf2[3] = swap[3]; + buf2[4] = swap[4]; + buf2[5] = swap[5]; + buf2[6] = swap[6]; + buf2[7] = swap[7]; +#endif + r = decoder(buf2); + ASSERT(r.val == num); + ASSERT(r.p == buf2 + upb_value_size(encoded)); + ASSERT(upb_zzenc_32(upb_zzdec_32(num)) == num); + } + + r = decoder(buf); + ASSERT(r.val == num); + ASSERT(r.p == buf + bytes); + ASSERT(upb_zzenc_64(upb_zzdec_64(num)) == num); +} + +static void test_varint_decoder(upb_decoderet (*decoder)(const char*)) { +#define TEST(bytes, expected_val) {\ + size_t n = sizeof(bytes) - 1; /* for NULL */ \ + char buf[UPB_PB_VARINT_MAX_LEN]; \ + upb_decoderet r; \ + memset(buf, 0xff, sizeof(buf)); \ + memcpy(buf, bytes, n); \ + r = decoder(buf); \ + ASSERT(r.val == expected_val); \ + ASSERT(r.p == buf + n); \ + } + + uint64_t num; + + char twelvebyte[16] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1}; + const char *twelvebyte_buf = twelvebyte; + /* A varint that terminates before hitting the end of the provided buffer, + * but in too many bytes (11 instead of 10). */ + upb_decoderet r = decoder(twelvebyte_buf); + ASSERT(r.p == NULL); + + TEST("\x00", 0ULL); + TEST("\x01", 1ULL); + TEST("\x81\x14", 0xa01ULL); + TEST("\x81\x03", 0x181ULL); + TEST("\x81\x83\x07", 0x1c181ULL); + TEST("\x81\x83\x87\x0f", 0x1e1c181ULL); + TEST("\x81\x83\x87\x8f\x1f", 0x1f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\x3f", 0x1f9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\x7f", 0x1fdf9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x01", 0x3fdf9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x81\x03", 0x303fdf9f1e1c181ULL); + TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x81\x83\x07", 0x8303fdf9f1e1c181ULL); +#undef TEST + + for (num = 5; num * 1.5 < UINT64_MAX; num *= 1.5) { + test_varint_for_num(decoder, num); + } + test_varint_for_num(decoder, 0); +} + + +#define TEST_VARINT_DECODER(decoder) \ + /* Create non-inline versions for convenient inspection of assembly language \ + * output. */ \ + upb_decoderet _upb_vdecode_ ## decoder(const char *p) { \ + return upb_vdecode_ ## decoder(p); \ + } \ + void test_ ## decoder(void) { \ + test_varint_decoder(&_upb_vdecode_ ## decoder); \ + } \ + +TEST_VARINT_DECODER(check2_branch32) +TEST_VARINT_DECODER(check2_branch64) + +int run_tests(int argc, char *argv[]) { + UPB_UNUSED(argc); + UPB_UNUSED(argv); + test_check2_branch32(); + test_check2_branch64(); + return 0; +} diff --git a/tests/test.proto b/tests/test.proto new file mode 100644 index 00000000000..e790cbf06d7 --- /dev/null +++ b/tests/test.proto @@ -0,0 +1,68 @@ + +// A series of messages with various kinds of cycles in them. +// +-+---+ +---+ +---+ +// V | | V | V | +// A -> B-+-> C -> D---+--->E---+ +// ^ |`---|--------^ +// +----------+----+ F + +syntax = "proto2"; + +message A { + optional B b = 1; +} + +message B { + optional B b = 1; + optional C c = 2; +} + +message C { + optional A a = 1; + optional B b = 2; + optional D d = 3; + optional E e = 4; +} + +message D { + optional A a = 1; + optional D d = 2; + optional E e = 3; +} + +message E { + optional E e = 1; +} + +message F { + optional E e = 1; +} + +// A proto with a bunch of simple primitives. +message SimplePrimitives { + optional fixed64 u64 = 1; + optional fixed32 u32 = 2; + optional double dbl = 3; + optional float flt = 5; + optional sint64 i64 = 6; + optional sint32 i32 = 7; + optional bool b = 8; + optional string str = 9; + + oneof foo { + int32 oneof_int32 = 10; + string oneof_string = 11; + } + + oneof bar { + int64 oneof_int64 = 13; + bytes oneof_bytes = 14; + } + + message Nested { + oneof foo { + int32 oneof_int32 = 10; + string b = 11; + } + } +} diff --git a/tests/test.proto.pb b/tests/test.proto.pb new file mode 100644 index 0000000000000000000000000000000000000000..a587fb18250872a4b66b976be4ed5efeafd5bde7 GIT binary patch literal 652 zcmZ{i-Acni5QUw@^k-U24G4=@8ZY#!LMwuoOZ(JH1ds6kH;Ne=(DVa0=l_ z!XkJ?7%`q#{-K08D*nN&1s-;`YX=h#X%Dp|hhjUgw}A8&df z(ZUC2{sp#k3F-CbRkYYmn%p?E@K+A?bVM3=3cDO?t~x6Epm!lUBxzTYklAU7wI_Nq ZD&C3z@2kV3mCZmDdUQca@?(kS={LWsf1v;X literal 0 HcmV?d00001 diff --git a/tests/test_cpp.cc b/tests/test_cpp.cc new file mode 100644 index 00000000000..abbafdabc8f --- /dev/null +++ b/tests/test_cpp.cc @@ -0,0 +1,957 @@ +/* + * + * Tests for C++ wrappers. + */ + +#include +#include + +#include +#include +#include +#include + +#include "tests/test_cpp.upbdefs.h" +#include "tests/upb_test.h" +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/pb/textprinter.h" +#include "upb/port_def.inc" +#include "upb/upb.h" + +template +void AssertInsert(T* const container, const typename T::value_type& val) { + bool inserted = container->insert(val).second; + ASSERT(inserted); +} + +// +// Tests for registering and calling handlers in all their variants. +// This test code is very repetitive because we have to declare each +// handler function variant separately, and they all have different +// signatures so it does not lend itself well to templates. +// +// We test three handler types: +// StartMessage (no data params) +// Int32 (1 data param (int32_t)) +// String Buf (2 data params (const char*, size_t)) +// +// For each handler type we test all 8 handler variants: +// (handler data?) x (function/method) x (returns {void, success}) +// +// The one notable thing we don't test at the moment is +// StartSequence/StartString handlers: these are different from StartMessage() +// in that they return void* for the sub-closure. But this is exercised in +// other tests. +// + +static const int kExpectedHandlerData = 1232323; + +class StringBufTesterBase { + public: + static const int kFieldNumber = 3; + + StringBufTesterBase() : seen_(false), handler_data_val_(0) {} + + void CallAndVerify(upb::Sink sink, upb::FieldDefPtr f) { + upb_selector_t start; + ASSERT(upb_handlers_getselector(f.ptr(), UPB_HANDLER_STARTSTR, &start)); + upb_selector_t str; + ASSERT(upb_handlers_getselector(f.ptr(), UPB_HANDLER_STRING, &str)); + + ASSERT(!seen_); + upb::Sink sub; + sink.StartMessage(); + sink.StartString(start, 0, &sub); + size_t ret = sub.PutStringBuffer(str, &buf_, 5, &handle_); + ASSERT(seen_); + ASSERT(len_ == 5); + ASSERT(ret == 5); + ASSERT(handler_data_val_ == kExpectedHandlerData); + } + + protected: + bool seen_; + int handler_data_val_; + size_t len_; + char buf_; + upb_bufhandle handle_; +}; + +// Test 8 combinations of: +// (handler data?) x (buffer handle?) x (function/method) +// +// Then we add one test each for this variation: to prevent combinatorial +// explosion of these tests we don't test the full 16 combinations, but +// rely on our knowledge that the implementation processes the return wrapping +// in a second separate and independent stage: +// +// (function/method) + +class StringBufTesterVoidMethodNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler(const char *buf, size_t len) { + ASSERT(buf == &buf_); + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidMethodNoHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodNoHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler(const char *buf, size_t len, const upb_bufhandle* handle) { + ASSERT(buf == &buf_); + ASSERT(handle == &handle_); + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidMethodWithHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodWithHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd, const char *buf, size_t len) { + ASSERT(buf == &buf_); + handler_data_val_ = *hd; + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidMethodWithHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidMethodWithHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd, const char* buf, size_t len, + const upb_bufhandle* handle) { + ASSERT(buf == &buf_); + ASSERT(handle == &handle_); + handler_data_val_ = *hd; + seen_ = true; + len_ = len; + } +}; + +class StringBufTesterVoidFunctionNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static void Handler(ME* t, const char *buf, size_t len) { + ASSERT(buf == &t->buf_); + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterVoidFunctionNoHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionNoHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static void Handler(ME* t, const char* buf, size_t len, + const upb_bufhandle* handle) { + ASSERT(buf == &t->buf_); + ASSERT(handle == &t->handle_); + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterVoidFunctionWithHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionWithHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd, const char *buf, size_t len) { + ASSERT(buf == &t->buf_); + t->handler_data_val_ = *hd; + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterVoidFunctionWithHandlerDataWithHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterVoidFunctionWithHandlerDataWithHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd, const char* buf, size_t len, + const upb_bufhandle* handle) { + ASSERT(buf == &t->buf_); + ASSERT(handle == &t->handle_); + t->handler_data_val_ = *hd; + t->seen_ = true; + t->len_ = len; + } +}; + +class StringBufTesterSizeTMethodNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterSizeTMethodNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + size_t Handler(const char *buf, size_t len) { + ASSERT(buf == &buf_); + seen_ = true; + len_ = len; + return len; + } +}; + +class StringBufTesterBoolMethodNoHandlerDataNoHandle + : public StringBufTesterBase { + public: + typedef StringBufTesterBoolMethodNoHandlerDataNoHandle ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStringHandler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + bool Handler(const char *buf, size_t len) { + ASSERT(buf == &buf_); + seen_ = true; + len_ = len; + return true; + } +}; + +class StartMsgTesterBase { + public: + // We don't need the FieldDef it will create, but the test harness still + // requires that we provide one. + static const int kFieldNumber = 3; + + StartMsgTesterBase() : seen_(false), handler_data_val_(0) {} + + void CallAndVerify(upb::Sink sink, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(!seen_); + sink.StartMessage(); + ASSERT(seen_); + ASSERT(handler_data_val_ == kExpectedHandlerData); + } + + protected: + bool seen_; + int handler_data_val_; +}; + +// Test all 8 combinations of: +// (handler data?) x (function/method) x (returns {void, bool}) + +class StartMsgTesterVoidFunctionNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + //static void Handler(ME* t) { + static void Handler(ME* t) { + t->seen_ = true; + } +}; + +class StartMsgTesterBoolFunctionNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static bool Handler(ME* t) { + t->seen_ = true; + return true; + } +}; + +class StartMsgTesterVoidMethodNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler() { + seen_ = true; + } +}; + +class StartMsgTesterBoolMethodNoHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + bool Handler() { + seen_ = true; + return true; + } +}; + +class StartMsgTesterVoidFunctionWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd) { + t->handler_data_val_ = *hd; + t->seen_ = true; + } +}; + +class StartMsgTesterBoolFunctionWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static bool Handler(ME* t, const int* hd) { + t->handler_data_val_ = *hd; + t->seen_ = true; + return true; + } +}; + +class StartMsgTesterVoidMethodWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterVoidMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd) { + handler_data_val_ = *hd; + seen_ = true; + } +}; + +class StartMsgTesterBoolMethodWithHandlerData : public StartMsgTesterBase { + public: + typedef StartMsgTesterBoolMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + UPB_UNUSED(f); + ASSERT(h.SetStartMessageHandler( + UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + bool Handler(const int* hd) { + handler_data_val_ = *hd; + seen_ = true; + return true; + } +}; + +class Int32ValueTesterBase { + public: + static const int kFieldNumber = 1; + + Int32ValueTesterBase() : seen_(false), val_(0), handler_data_val_(0) {} + + void CallAndVerify(upb::Sink sink, upb::FieldDefPtr f) { + upb_selector_t s; + ASSERT(upb_handlers_getselector(f.ptr(), UPB_HANDLER_INT32, &s)); + + ASSERT(!seen_); + sink.PutInt32(s, 5); + ASSERT(seen_); + ASSERT(handler_data_val_ == kExpectedHandlerData); + ASSERT(val_ == 5); + } + + protected: + bool seen_; + int32_t val_; + int handler_data_val_; +}; + +// Test all 8 combinations of: +// (handler data?) x (function/method) x (returns {void, bool}) + +class ValueTesterInt32VoidFunctionNoHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static void Handler(ME* t, int32_t val) { + t->val_ = val; + t->seen_ = true; + } +}; + +class ValueTesterInt32BoolFunctionNoHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolFunctionNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + static bool Handler(ME* t, int32_t val) { + t->val_ = val; + t->seen_ = true; + return true; + } +}; + +class ValueTesterInt32VoidMethodNoHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + void Handler(int32_t val) { + val_ = val; + seen_ = true; + } +}; + +class ValueTesterInt32BoolMethodNoHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolMethodNoHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler(f, UpbMakeHandler(&ME::Handler))); + handler_data_val_ = kExpectedHandlerData; + } + + private: + bool Handler(int32_t val) { + val_ = val; + seen_ = true; + return true; + } +}; + +class ValueTesterInt32VoidFunctionWithHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static void Handler(ME* t, const int* hd, int32_t val) { + t->val_ = val; + t->handler_data_val_ = *hd; + t->seen_ = true; + } +}; + +class ValueTesterInt32BoolFunctionWithHandlerData + : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolFunctionWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&Handler, new int(kExpectedHandlerData)))); + } + + private: + static bool Handler(ME* t, const int* hd, int32_t val) { + t->val_ = val; + t->handler_data_val_ = *hd; + t->seen_ = true; + return true; + } +}; + +class ValueTesterInt32VoidMethodWithHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32VoidMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + void Handler(const int* hd, int32_t val) { + val_ = val; + handler_data_val_ = *hd; + seen_ = true; + } +}; + +class ValueTesterInt32BoolMethodWithHandlerData : public Int32ValueTesterBase { + public: + typedef ValueTesterInt32BoolMethodWithHandlerData ME; + void Register(upb::HandlersPtr h, upb::FieldDefPtr f) { + ASSERT(h.SetInt32Handler( + f, UpbBind(&ME::Handler, new int(kExpectedHandlerData)))); + } + + private: + bool Handler(const int* hd, int32_t val) { + val_ = val; + handler_data_val_ = *hd; + seen_ = true; + return true; + } +}; + +template +void RegisterHandlers(const void* closure, upb::Handlers* h_ptr) { + T* tester = const_cast(static_cast(closure)); + upb::HandlersPtr h(h_ptr); + upb::FieldDefPtr f = h.message_def().FindFieldByNumber(T::kFieldNumber); + ASSERT(f); + tester->Register(h, f); +} + +template +void TestHandler() { + T tester; + upb::SymbolTable symtab; + upb::HandlerCache cache(&RegisterHandlers, &tester); + upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + ASSERT(md); + upb::FieldDefPtr f = md.FindFieldByNumber(T::kFieldNumber); + ASSERT(f); + + const upb::Handlers* h = cache.Get(md); + + upb::Sink sink(h, &tester); + tester.CallAndVerify(sink, f); +} + +class T1 {}; +class T2 {}; + +template +void DoNothingHandler(C* closure) { + UPB_UNUSED(closure); +} + +template +void DoNothingInt32Handler(C* closure, int32_t val) { + UPB_UNUSED(closure); + UPB_UNUSED(val); +} + +template +class DoNothingStartHandler { + public: + // We wrap these functions inside of a class for a somewhat annoying reason. + // UpbMakeHandler() is a macro, so we can't say + // UpbMakeHandler(DoNothingStartHandler) + // + // because otherwise the preprocessor gets confused at the comma and tries to + // make it two macro arguments. The usual solution doesn't work either: + // UpbMakeHandler((DoNothingStartHandler)) + // + // If we do that the macro expands correctly, but then it tries to pass that + // parenthesized expression as a template parameter, ie. Type<(F)>, which + // isn't legal C++ (Clang will compile it but complains with + // warning: address non-type template argument cannot be surrounded by + // parentheses + // + // This two-level thing allows us to effectively pass two template parameters, + // but without any commas: + // UpbMakeHandler(DoNothingStartHandler::Handler) + template + static R* Handler(C* closure) { + UPB_UNUSED(closure); + return NULL; + } + + template + static R* String(C* closure, size_t size_len) { + UPB_UNUSED(closure); + UPB_UNUSED(size_len); + return NULL; + } +}; + +template +void DoNothingStringBufHandler(C* closure, const char *buf, size_t len) { + UPB_UNUSED(closure); + UPB_UNUSED(buf); + UPB_UNUSED(len); +} + +template +void DoNothingEndMessageHandler(C* closure, upb_status *status) { + UPB_UNUSED(closure); + UPB_UNUSED(status); +} + +void RegisterMismatchedTypes(const void* closure, upb::Handlers* h_ptr) { + upb::HandlersPtr h(h_ptr); + + upb::MessageDefPtr md(h.message_def()); + ASSERT(md); + upb::FieldDefPtr i32 = md.FindFieldByName("i32"); + upb::FieldDefPtr r_i32 = md.FindFieldByName("r_i32"); + upb::FieldDefPtr str = md.FindFieldByName("str"); + upb::FieldDefPtr r_str = md.FindFieldByName("r_str"); + upb::FieldDefPtr msg = md.FindFieldByName("msg"); + upb::FieldDefPtr r_msg = md.FindFieldByName("r_msg"); + ASSERT(i32); + ASSERT(r_i32); + ASSERT(str); + ASSERT(r_str); + ASSERT(msg); + ASSERT(r_msg); + + // Establish T1 as the top-level closure type. + ASSERT(h.SetInt32Handler(i32, UpbMakeHandler(DoNothingInt32Handler))); + + // Now any other attempt to set another handler with T2 as the top-level + // closure should fail. But setting these same handlers with T1 as the + // top-level closure will succeed. + ASSERT(!h.SetStartMessageHandler(UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetStartMessageHandler(UpbMakeHandler(DoNothingHandler))); + + ASSERT( + !h.SetEndMessageHandler(UpbMakeHandler(DoNothingEndMessageHandler))); + ASSERT( + h.SetEndMessageHandler(UpbMakeHandler(DoNothingEndMessageHandler))); + + ASSERT(!h.SetStartStringHandler( + str, UpbMakeHandler(DoNothingStartHandler::String))); + ASSERT(h.SetStartStringHandler( + str, UpbMakeHandler(DoNothingStartHandler::String))); + + ASSERT(!h.SetEndStringHandler(str, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndStringHandler(str, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSubMessageHandler( + msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSubMessageHandler( + msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT( + !h.SetEndSubMessageHandler(msg, UpbMakeHandler(DoNothingHandler))); + ASSERT( + h.SetEndSubMessageHandler(msg, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSequenceHandler( + r_i32, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSequenceHandler( + r_i32, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSequenceHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSequenceHandler( + r_msg, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSequenceHandler( + r_msg, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSequenceHandler( + r_str, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSequenceHandler( + r_str, UpbMakeHandler(DoNothingHandler))); + + // By setting T1 as the return type for the Start* handlers we have + // established T1 as the type of the sequence and string frames. + // Setting callbacks that use T2 should fail, but T1 should succeed. + ASSERT( + !h.SetStringHandler(str, UpbMakeHandler(DoNothingStringBufHandler))); + ASSERT( + h.SetStringHandler(str, UpbMakeHandler(DoNothingStringBufHandler))); + + ASSERT(!h.SetInt32Handler(r_i32, UpbMakeHandler(DoNothingInt32Handler))); + ASSERT(h.SetInt32Handler(r_i32, UpbMakeHandler(DoNothingInt32Handler))); + + ASSERT(!h.SetStartSubMessageHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSubMessageHandler( + r_msg, UpbMakeHandler(DoNothingStartHandler::Handler))); + + ASSERT(!h.SetEndSubMessageHandler(r_msg, + UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndSubMessageHandler(r_msg, + UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + ASSERT(h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + + ASSERT( + !h.SetEndStringHandler(r_str, UpbMakeHandler(DoNothingHandler))); + ASSERT(h.SetEndStringHandler(r_str, UpbMakeHandler(DoNothingHandler))); + + ASSERT(!h.SetStringHandler(r_str, + UpbMakeHandler(DoNothingStringBufHandler))); + ASSERT(h.SetStringHandler(r_str, + UpbMakeHandler(DoNothingStringBufHandler))); +} + +void RegisterMismatchedTypes2(const void* closure, upb::Handlers* h_ptr) { + upb::HandlersPtr h(h_ptr); + + upb::MessageDefPtr md(h.message_def()); + ASSERT(md); + upb::FieldDefPtr i32 = md.FindFieldByName("i32"); + upb::FieldDefPtr r_i32 = md.FindFieldByName("r_i32"); + upb::FieldDefPtr str = md.FindFieldByName("str"); + upb::FieldDefPtr r_str = md.FindFieldByName("r_str"); + upb::FieldDefPtr msg = md.FindFieldByName("msg"); + upb::FieldDefPtr r_msg = md.FindFieldByName("r_msg"); + ASSERT(i32); + ASSERT(r_i32); + ASSERT(str); + ASSERT(r_str); + ASSERT(msg); + ASSERT(r_msg); + + // For our second test we do the same in reverse. We directly set the type of + // the frame and then observe failures at registering a Start* handler that + // returns a different type. + + // First establish the type of a sequence frame directly. + ASSERT(h.SetInt32Handler(r_i32, UpbMakeHandler(DoNothingInt32Handler))); + + // Now setting a StartSequence callback that returns a different type should + // fail. + ASSERT(!h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_i32, UpbMakeHandler(DoNothingStartHandler::Handler))); + + // Establish a string frame directly. + ASSERT(h.SetStringHandler(r_str, + UpbMakeHandler(DoNothingStringBufHandler))); + + // Fail setting a StartString callback that returns a different type. + ASSERT(!h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + ASSERT(h.SetStartStringHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::String))); + + // The previous established T1 as the frame for the r_str sequence. + ASSERT(!h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); + ASSERT(h.SetStartSequenceHandler( + r_str, UpbMakeHandler(DoNothingStartHandler::Handler))); +} + +void TestMismatchedTypes() { + // First create a schema for our test. + upb::SymbolTable symtab; + upb::HandlerCache handler_cache(&RegisterMismatchedTypes, nullptr); + upb::HandlerCache handler_cache2(&RegisterMismatchedTypes2, nullptr); + const upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + + // Now test the type-checking in handler registration. + handler_cache.Get(md); + handler_cache2.Get(md); +} + +class IntIncrementer { + public: + explicit IntIncrementer(int* x) : x_(x) { (*x_)++; } + ~IntIncrementer() { (*x_)--; } + + static void Handler(void* closure, const IntIncrementer* incrementer, + int32_t x) { + UPB_UNUSED(closure); + UPB_UNUSED(incrementer); + UPB_UNUSED(x); + } + + private: + int* x_; +}; + +void RegisterIncrementor(const void* closure, upb::Handlers* h_ptr) { + const int* x = static_cast(closure); + upb::HandlersPtr h(h_ptr); + upb::FieldDefPtr f = h.message_def().FindFieldByName("i32"); + h.SetInt32Handler(f, UpbBind(&IntIncrementer::Handler, + new IntIncrementer(const_cast(x)))); +} + +void TestHandlerDataDestruction() { + int x = 0; + + { + upb::SymbolTable symtab; + upb::HandlerCache cache(&RegisterIncrementor, &x); + upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + cache.Get(md); + ASSERT(x == 1); + } + + ASSERT(x == 0); +} + +void TestIteration() { + upb::SymbolTable symtab; + upb::MessageDefPtr md(upb_test_TestMessage_getmsgdef(symtab.ptr())); + + // Test range-based for on both fields and oneofs (with the iterator adaptor). + int field_count = 0; + for (auto field : md.fields()) { + UPB_UNUSED(field); + field_count++; + } + ASSERT(field_count == md.field_count()); + + int oneof_count = 0; + for (auto oneof : md.oneofs()) { + UPB_UNUSED(oneof); + oneof_count++; + } + ASSERT(oneof_count == md.oneof_count()); +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + TestHandler(); + + TestMismatchedTypes(); + + TestHandlerDataDestruction(); + TestIteration(); + + return 0; +} + +} diff --git a/tests/test_cpp.proto b/tests/test_cpp.proto new file mode 100644 index 00000000000..f890350ece1 --- /dev/null +++ b/tests/test_cpp.proto @@ -0,0 +1,12 @@ +syntax = "proto2"; + +package upb.test; + +message TestMessage { + optional int32 i32 = 1; + repeated int32 r_i32 = 2; + optional string str = 3; + repeated string r_str = 4; + optional TestMessage msg = 5; + repeated TestMessage r_msg = 6; +} diff --git a/tests/test_table.cc b/tests/test_table.cc new file mode 100644 index 00000000000..063eb68cee6 --- /dev/null +++ b/tests/test_table.cc @@ -0,0 +1,679 @@ +/* + * + * Tests for upb_table. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tests/upb_test.h" +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +// Convenience interface for C++. We don't put this in upb itself because +// the table is not exposed to users. + +namespace upb { + +template upb_value MakeUpbValue(T val); +template T GetUpbValue(upb_value val); +template upb_ctype_t GetUpbValueType(); + +#define FUNCS(name, type_t, enumval) \ + template<> upb_value MakeUpbValue(type_t val) { return upb_value_ ## name(val); } \ + template<> type_t GetUpbValue(upb_value val) { return upb_value_get ## name(val); } \ + template<> upb_ctype_t GetUpbValueType() { return enumval; } + +FUNCS(int32, int32_t, UPB_CTYPE_INT32) +FUNCS(int64, int64_t, UPB_CTYPE_INT64) +FUNCS(uint32, uint32_t, UPB_CTYPE_UINT32) +FUNCS(uint64, uint64_t, UPB_CTYPE_UINT64) +FUNCS(bool, bool, UPB_CTYPE_BOOL) +FUNCS(cstr, char*, UPB_CTYPE_CSTR) +FUNCS(ptr, void*, UPB_CTYPE_PTR) +FUNCS(constptr, const void*, UPB_CTYPE_CONSTPTR) +FUNCS(fptr, upb_func*, UPB_CTYPE_FPTR) + +#undef FUNCS + +class IntTable { + public: + IntTable(upb_ctype_t value_type) { upb_inttable_init(&table_, value_type); } + ~IntTable() { upb_inttable_uninit(&table_); } + + size_t count() { return upb_inttable_count(&table_); } + + bool Insert(uintptr_t key, upb_value val) { + return upb_inttable_insert(&table_, key, val); + } + + bool Replace(uintptr_t key, upb_value val) { + return upb_inttable_replace(&table_, key, val); + } + + std::pair Remove(uintptr_t key) { + std::pair ret; + ret.first = upb_inttable_remove(&table_, key, &ret.second); + return ret; + } + + std::pair Lookup(uintptr_t key) const { + std::pair ret; + ret.first = upb_inttable_lookup(&table_, key, &ret.second); + return ret; + } + + std::pair Lookup32(uint32_t key) const { + std::pair ret; + ret.first = upb_inttable_lookup32(&table_, key, &ret.second); + return ret; + } + + void Compact() { upb_inttable_compact(&table_); } + + class iterator : public std::iterator > { + public: + explicit iterator(IntTable* table) { + upb_inttable_begin(&iter_, &table->table_); + } + + static iterator end(IntTable* table) { + iterator iter(table); + upb_inttable_iter_setdone(&iter.iter_); + return iter; + } + + void operator++() { + return upb_inttable_next(&iter_); + } + + std::pair operator*() const { + std::pair ret; + ret.first = upb_inttable_iter_key(&iter_); + ret.second = upb_inttable_iter_value(&iter_); + return ret; + } + + bool operator==(const iterator& other) const { + return upb_inttable_iter_isequal(&iter_, &other.iter_); + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + + private: + upb_inttable_iter iter_; + }; + + upb_inttable table_; +}; + +class StrTable { + public: + StrTable(upb_ctype_t value_type) { upb_strtable_init(&table_, value_type); } + ~StrTable() { upb_strtable_uninit(&table_); } + + size_t count() { return upb_strtable_count(&table_); } + + bool Insert(const std::string& key, upb_value val) { + return upb_strtable_insert2(&table_, key.c_str(), key.size(), val); + } + + std::pair Remove(const std::string& key) { + std::pair ret; + ret.first = + upb_strtable_remove2(&table_, key.c_str(), key.size(), &ret.second); + return ret; + } + + std::pair Lookup(const std::string& key) const { + std::pair ret; + ret.first = + upb_strtable_lookup2(&table_, key.c_str(), key.size(), &ret.second); + return ret; + } + + void Resize(size_t size_lg2) { + upb_strtable_resize(&table_, size_lg2, &upb_alloc_global); + } + + class iterator : public std::iterator > { + public: + explicit iterator(StrTable* table) { + upb_strtable_begin(&iter_, &table->table_); + } + + static iterator end(StrTable* table) { + iterator iter(table); + upb_strtable_iter_setdone(&iter.iter_); + return iter; + } + + void operator++() { + return upb_strtable_next(&iter_); + } + + std::pair operator*() const { + std::pair ret; + ret.first.assign(upb_strtable_iter_key(&iter_)); + ret.second = upb_strtable_iter_value(&iter_); + return ret; + } + + bool operator==(const iterator& other) const { + return upb_strtable_iter_isequal(&iter_, &other.iter_); + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + + private: + upb_strtable_iter iter_; + }; + + upb_strtable table_; +}; + +template class TypedStrTable { + public: + TypedStrTable() : table_(GetUpbValueType()) {} + + size_t count() { return table_.count(); } + + bool Insert(const std::string &key, T val) { + return table_.Insert(key, MakeUpbValue(val)); + } + + std::pair Remove(const std::string& key) { + std::pair found = table_.Remove(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + std::pair Lookup(const std::string& key) const { + std::pair found = table_.Lookup(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + void Resize(size_t size_lg2) { + table_.Resize(size_lg2); + } + + class iterator : public std::iterator > { + public: + explicit iterator(TypedStrTable* table) : iter_(&table->table_) {} + static iterator end(TypedStrTable* table) { + iterator iter(table); + iter.iter_ = StrTable::iterator::end(&table->table_); + return iter; + } + + void operator++() { ++iter_; } + + std::pair operator*() const { + std::pair val = *iter_; + std::pair ret; + ret.first = val.first; + ret.second = GetUpbValue(val.second); + return ret; + } + + bool operator==(const iterator& other) const { + return iter_ == other.iter_; + } + + bool operator!=(const iterator& other) const { + return iter_ != other.iter_; + } + + private: + StrTable::iterator iter_; + }; + + iterator begin() { return iterator(this); } + iterator end() { return iterator::end(this); } + + StrTable table_; +}; + +template class TypedIntTable { + public: + TypedIntTable() : table_(GetUpbValueType()) {} + + size_t count() { return table_.count(); } + + bool Insert(uintptr_t key, T val) { + return table_.Insert(key, MakeUpbValue(val)); + } + + bool Replace(uintptr_t key, T val) { + return table_.Replace(key, MakeUpbValue(val)); + } + + std::pair Remove(uintptr_t key) { + std::pair found = table_.Remove(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + std::pair Lookup(uintptr_t key) const { + std::pair found = table_.Lookup(key); + std::pair ret; + ret.first = found.first; + if (ret.first) { + ret.second = GetUpbValue(found.second); + } + return ret; + } + + void Compact() { table_.Compact(); } + + class iterator : public std::iterator > { + public: + explicit iterator(TypedIntTable* table) : iter_(&table->table_) {} + static iterator end(TypedIntTable* table) { + return IntTable::iterator::end(&table->table_); + } + + void operator++() { ++iter_; } + + std::pair operator*() const { + std::pair val = *iter_; + std::pair ret; + ret.first = val.first; + ret.second = GetUpbValue(val.second); + return ret; + } + + bool operator==(const iterator& other) const { + return iter_ == other.iter_; + } + + bool operator!=(const iterator& other) const { + return iter_ != other.iter_; + } + + private: + IntTable::iterator iter_; + }; + + iterator begin() { return iterator(this); } + iterator end() { return iterator::end(this); } + + IntTable table_; +}; + +} + +bool benchmark = false; +#define CPU_TIME_PER_TEST 0.5 + +using std::vector; + +double get_usertime() { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec + (usage.ru_utime.tv_usec/1000000.0); +} + +/* num_entries must be a power of 2. */ +void test_strtable(const vector& keys, uint32_t num_to_insert) { + /* Initialize structures. */ + std::map m; + typedef upb::TypedStrTable Table; + Table table; + std::set all; + for(size_t i = 0; i < num_to_insert; i++) { + const std::string& key = keys[i]; + all.insert(key); + table.Insert(key, key[0]); + m[key] = key[0]; + } + + /* Test correctness. */ + for(uint32_t i = 0; i < keys.size(); i++) { + const std::string& key = keys[i]; + std::pair found = table.Lookup(key); + if(m.find(key) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == key[0]); + ASSERT(m[key] == key[0]); + } else { + ASSERT(!found.first); + } + } + + for (Table::iterator it = table.begin(); it != table.end(); ++it) { + std::set::iterator i = all.find((*it).first); + ASSERT(i != all.end()); + all.erase(i); + } + ASSERT(all.empty()); + + // Test iteration with resizes. + + for (int i = 0; i < 10; i++) { + for (Table::iterator it = table.begin(); it != table.end(); ++it) { + // Even if we invalidate the iterator it should only return real elements. + ASSERT((*it).second == m[(*it).first]); + + // Force a resize even though the size isn't changing. + // Also forces the table size to grow so some new buckets end up empty. + int new_lg2 = table.table_.table_.t.size_lg2 + 1; + // Don't use more than 64k tables, to avoid exhausting memory. + new_lg2 = UPB_MIN(new_lg2, 16); + table.Resize(new_lg2); + } + } + +} + +/* num_entries must be a power of 2. */ +void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { + /* Initialize structures. */ + typedef upb::TypedIntTable Table; + Table table; + uint32_t largest_key = 0; + std::map m; + std::unordered_map hm; + for(size_t i = 0; i < num_entries; i++) { + int32_t key = keys[i]; + largest_key = UPB_MAX((int32_t)largest_key, key); + table.Insert(key, key * 2); + m[key] = key*2; + hm[key] = key*2; + } + + /* Test correctness. */ + for(uint32_t i = 0; i <= largest_key; i++) { + std::pair found = table.Lookup(i); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == i*2); + ASSERT(m[i] == i*2); + ASSERT(hm[i] == i*2); + } else { + ASSERT(!found.first); + } + } + + for(uint16_t i = 0; i < num_entries; i += 2) { + std::pair found = table.Remove(keys[i]); + ASSERT(found.first == (m.erase(keys[i]) == 1)); + if (found.first) ASSERT(found.second == (uint32_t)keys[i] * 2); + hm.erase(keys[i]); + m.erase(keys[i]); + } + + ASSERT(table.count() == hm.size()); + + /* Test correctness. */ + for(uint32_t i = 0; i <= largest_key; i++) { + std::pair found = table.Lookup(i); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == i*2); + ASSERT(m[i] == i*2); + ASSERT(hm[i] == i*2); + } else { + ASSERT(!found.first); + } + } + + // Test replace. + for(uint32_t i = 0; i <= largest_key; i++) { + bool replaced = table.Replace(i, i*3); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(replaced); + m[i] = i * 3; + hm[i] = i * 3; + } else { + ASSERT(!replaced); + } + } + + // Compact and test correctness again. + table.Compact(); + for(uint32_t i = 0; i <= largest_key; i++) { + std::pair found = table.Lookup(i); + if(m.find(i) != m.end()) { /* Assume map implementation is correct. */ + ASSERT(found.first); + ASSERT(found.second == i*3); + ASSERT(m[i] == i*3); + ASSERT(hm[i] == i*3); + } else { + ASSERT(!found.first); + } + } + + if(!benchmark) { + return; + } + + printf("%s\n", desc); + + /* Test performance. We only test lookups for keys that are known to exist. */ + uint16_t *rand_order = new uint16_t[num_entries]; + for(uint16_t i = 0; i < num_entries; i++) { + rand_order[i] = i; + } + for(uint16_t i = num_entries - 1; i >= 1; i--) { + uint16_t rand_i = (random() / (double)RAND_MAX) * i; + ASSERT(rand_i <= i); + uint16_t tmp = rand_order[rand_i]; + rand_order[rand_i] = rand_order[i]; + rand_order[i] = tmp; + } + + uintptr_t x = 0; + const int mask = num_entries - 1; + int time_mask = 0xffff; + + printf("upb_inttable(seq): "); + fflush(stdout); + double before = get_usertime(); + unsigned int i; + +#define MAYBE_BREAK \ + if ((i & time_mask) == 0 && (get_usertime() - before) > CPU_TIME_PER_TEST) \ + break; + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[i & mask]; + upb_value v; + bool ok = upb_inttable_lookup32(&table.table_.table_, key, &v); + x += (uintptr_t)ok; + } + double total = get_usertime() - before; + printf("%ld/s\n", (long)(i/total)); + double upb_seq_i = i / 100; // For later percentage calcuation. + + printf("upb_inttable(rand): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + upb_value v; + bool ok = upb_inttable_lookup32(&table.table_.table_, key, &v); + x += (uintptr_t)ok; + } + total = get_usertime() - before; + printf("%ld/s\n", (long)(i/total)); + double upb_rand_i = i / 100; // For later percentage calculation. + + printf("std::map(seq): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[i & mask]; + x += m[key]; + } + total = get_usertime() - before; + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_seq_i); + + printf("std::map(rand): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + x += m[key]; + } + total = get_usertime() - before; + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_rand_i); + + printf("std::unordered_map(seq): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + x += hm[key]; + } + total = get_usertime() - before; + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_seq_i); + + printf("std::unordered_map(rand): "); + fflush(stdout); + before = get_usertime(); + for(i = 0; true; i++) { + MAYBE_BREAK; + int32_t key = keys[rand_order[i & mask]]; + x += hm[key]; + } + total = get_usertime() - before; + if (x == INT_MAX) abort(); + printf("%ld/s (%0.1f%% of upb)\n\n", (long)(i/total), i / upb_rand_i); + delete[] rand_order; +} + +/* + * This test can't pass right now because the table can't store a value of + * (uint64_t)-1. + */ +void test_int64_max_value() { +/* + typedef upb::TypedIntTable Table; + Table table; + uintptr_t uint64_max = (uint64_t)-1; + table.Insert(1, uint64_max); + std::pair found = table.Lookup(1); + ASSERT(found.first); + ASSERT(found.second == uint64_max); +*/ +} + +int32_t *get_contiguous_keys(int32_t num) { + int32_t *buf = new int32_t[num]; + for(int32_t i = 0; i < num; i++) + buf[i] = i; + return buf; +} + +void test_delete() { + upb_inttable t; + upb_inttable_init(&t, UPB_CTYPE_BOOL); + upb_inttable_insert(&t, 0, upb_value_bool(true)); + upb_inttable_insert(&t, 2, upb_value_bool(true)); + upb_inttable_insert(&t, 4, upb_value_bool(true)); + upb_inttable_compact(&t); + upb_inttable_remove(&t, 0, NULL); + upb_inttable_remove(&t, 2, NULL); + upb_inttable_remove(&t, 4, NULL); + + upb_inttable_iter iter; + for (upb_inttable_begin(&iter, &t); !upb_inttable_done(&iter); + upb_inttable_next(&iter)) { + ASSERT(false); + } + + upb_inttable_uninit(&t); +} + +extern "C" { + +int run_tests(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "benchmark") == 0) benchmark = true; + } + + vector keys; + keys.push_back("google.protobuf.FileDescriptorSet"); + keys.push_back("google.protobuf.FileDescriptorProto"); + keys.push_back("google.protobuf.DescriptorProto"); + keys.push_back("google.protobuf.DescriptorProto.ExtensionRange"); + keys.push_back("google.protobuf.FieldDescriptorProto"); + keys.push_back("google.protobuf.EnumDescriptorProto"); + keys.push_back("google.protobuf.EnumValueDescriptorProto"); + keys.push_back("google.protobuf.ServiceDescriptorProto"); + keys.push_back("google.protobuf.MethodDescriptorProto"); + keys.push_back("google.protobuf.FileOptions"); + keys.push_back("google.protobuf.MessageOptions"); + keys.push_back("google.protobuf.FieldOptions"); + keys.push_back("google.protobuf.EnumOptions"); + keys.push_back("google.protobuf.EnumValueOptions"); + keys.push_back("google.protobuf.ServiceOptions"); + keys.push_back("google.protobuf.MethodOptions"); + keys.push_back("google.protobuf.UninterpretedOption"); + keys.push_back("google.protobuf.UninterpretedOption.NamePart"); + + for (int i = 0; i < 10; i++) { + test_strtable(keys, 18); + } + + int32_t *keys1 = get_contiguous_keys(8); + test_inttable(keys1, 8, "Table size: 8, keys: 1-8 ===="); + delete[] keys1; + + int32_t *keys2 = get_contiguous_keys(64); + test_inttable(keys2, 64, "Table size: 64, keys: 1-64 ====\n"); + delete[] keys2; + + int32_t *keys3 = get_contiguous_keys(512); + test_inttable(keys3, 512, "Table size: 512, keys: 1-512 ====\n"); + delete[] keys3; + + int32_t *keys4 = new int32_t[64]; + for(int32_t i = 0; i < 64; i++) { + if(i < 32) + keys4[i] = i+1; + else + keys4[i] = 10101+i; + } + test_inttable(keys4, 64, "Table size: 64, keys: 1-32 and 10133-10164 ====\n"); + delete[] keys4; + + test_delete(); + test_int64_max_value(); + + return 0; +} + +} diff --git a/tests/test_util.h b/tests/test_util.h new file mode 100644 index 00000000000..485410082df --- /dev/null +++ b/tests/test_util.h @@ -0,0 +1,230 @@ +/* +** Common functionality for tests. +**/ + +#ifndef UPB_TEST_UTIL_H_ +#define UPB_TEST_UTIL_H_ + +#include +#include +#include "tests/upb_test.h" +#include "upb/sink.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus + +upb_bufhandle global_handle; + +/* A convenience class for parser tests. Provides some useful features: + * + * - can support multiple calls to parse, to test the parser's handling + * of buffer seams. + * + * - can output verbose output about each parse call when requested, for + * ease of debugging. + * + * - can pass NULL for skipped regions of the input if requested. + * + * - allocates and passes a separate buffer for each parsed region, to + * ensure that the parser is not erroneously overreading its buffer. + */ +class VerboseParserEnvironment { + public: + /* Pass verbose=true to print detailed diagnostics to stderr. */ + VerboseParserEnvironment(bool verbose) : verbose_(verbose) {} + + void Reset(const char *buf, size_t len, bool may_skip, bool expect_error) { + buf_ = buf; + len_ = len; + ofs_ = 0; + expect_error_ = expect_error; + end_ok_set_ = false; + skip_until_ = may_skip ? 0 : -1; + skipped_with_null_ = false; + } + + /* The user should call a series of: + * + * Reset(buf, len, may_skip); + * Start() + * ParseBuffer(X); + * ParseBuffer(Y); + * // Repeat ParseBuffer as desired, but last call should pass -1. + * ParseBuffer(-1); + * End(); + */ + + + bool Start() { + if (verbose_) { + fprintf(stderr, "Calling start()\n"); + } + return sink_.Start(len_, &subc_); + } + + bool End() { + if (verbose_) { + fprintf(stderr, "Calling end()\n"); + } + end_ok_ = sink_.End(); + end_ok_set_ = true; + + return end_ok_; + } + + bool CheckConsistency() { + /* If we called end (which we should only do when previous bytes are fully + * accepted), then end() should return true iff there were no errors. */ + if (end_ok_set_ && end_ok_ != status_.ok()) { + fprintf(stderr, "End() status and saw_error didn't match.\n"); + return false; + } + + if (expect_error_ && status_.ok()) { + fprintf(stderr, "Expected error but saw none.\n"); + return false; + } + + if (!status_.ok()) { + if (expect_error_ && verbose_) { + fprintf(stderr, "Encountered error, as expected: %s", + status_.error_message()); + } else if (!expect_error_) { + fprintf(stderr, "Encountered unexpected error: %s", + status_.error_message()); + return false; + } + } + + return true; + } + + bool ParseBuffer(int bytes) { + if (bytes < 0) { + bytes = len_ - ofs_; + } + + ASSERT((size_t)bytes <= (len_ - ofs_)); + + /* Copy buffer into a separate, temporary buffer. + * This is necessary to verify that the parser is not erroneously + * reading outside the specified bounds. */ + char *buf2 = NULL; + + if ((int)(ofs_ + bytes) <= skip_until_) { + skipped_with_null_ = true; + } else { + buf2 = (char*)malloc(bytes); + UPB_ASSERT(buf2); + memcpy(buf2, buf_ + ofs_, bytes); + } + + if (buf2 == NULL && bytes == 0) { + /* Decoders dont' support buf=NULL, bytes=0. */ + return true; + } + + if (verbose_) { + fprintf(stderr, "Calling parse(%u) for bytes %u-%u of the input\n", + (unsigned)bytes, (unsigned)ofs_, (unsigned)(ofs_ + bytes)); + } + + int parsed = sink_.PutBuffer(subc_, buf2, bytes, &global_handle); + free(buf2); + + if (verbose_) { + if (parsed == bytes) { + fprintf(stderr, + "parse(%u) = %u, complete byte count indicates success\n", + (unsigned)bytes, (unsigned)bytes); + } else if (parsed > bytes) { + fprintf(stderr, + "parse(%u) = %u, long byte count indicates success and skip " + "of the next %u bytes\n", + (unsigned)bytes, (unsigned)parsed, (unsigned)(parsed - bytes)); + } else { + fprintf(stderr, + "parse(%u) = %u, short byte count indicates failure; " + "last %u bytes were not consumed\n", + (unsigned)bytes, (unsigned)parsed, (unsigned)(bytes - parsed)); + } + } + + if (!status_.ok()) { + return false; + } + + if (parsed > bytes && skip_until_ >= 0) { + skip_until_ = ofs_ + parsed; + } + + ofs_ += UPB_MIN(parsed, bytes); + + return true; + } + + void ResetBytesSink(upb::BytesSink sink) { + sink_ = sink; + } + + size_t ofs() { return ofs_; } + + bool SkippedWithNull() { return skipped_with_null_; } + + upb::Arena* arena() { return &arena_; } + upb::Status* status() { return &status_; } + + private: + upb::Arena arena_; + upb::Status status_; + upb::BytesSink sink_; + const char* buf_; + size_t len_; + bool verbose_; + size_t ofs_; + void *subc_; + bool expect_error_; + bool end_ok_; + bool end_ok_set_; + + /* When our parse call returns a value greater than the number of bytes + * we passed in, the decoder is indicating to us that the next N bytes + * in the stream are not needed and can be skipped. The user is allowed + * to pass a NULL buffer for those N bytes. + * + * skip_until_ is initially set to 0 if we should do this NULL-buffer + * skipping or -1 if we should not. If we are open to doing NULL-buffer + * skipping and we get an opportunity to do it, we set skip_until to the + * stream offset where we can skip until. The user can then test whether + * this happened by testing SkippedWithNull(). */ + int skip_until_; + bool skipped_with_null_; +}; + +#endif /* __cplusplus */ + +UPB_INLINE char *upb_readfile(const char *filename, size_t *len) { + long size; + char *buf; + FILE *f = fopen(filename, "rb"); + if(!f) return NULL; + if(fseek(f, 0, SEEK_END) != 0) goto error; + size = ftell(f); + if(size < 0) goto error; + if(fseek(f, 0, SEEK_SET) != 0) goto error; + buf = (char*)malloc(size + 1); + if(size && fread(buf, size, 1, f) != 1) goto error; + fclose(f); + if (len) *len = size; + buf[size] = '\0'; + return buf; + +error: + fclose(f); + return NULL; +} + +#include "upb/port_undef.inc" + +#endif /* UPB_TEST_UTIL_H_ */ diff --git a/tests/testmain.cc b/tests/testmain.cc new file mode 100644 index 00000000000..97dd7169a46 --- /dev/null +++ b/tests/testmain.cc @@ -0,0 +1,16 @@ + +#include +#ifdef USE_GOOGLE +#include "base/init_google.h" +#endif + +extern "C" { +int run_tests(int argc, char *argv[]); +} + +int main(int argc, char *argv[]) { +#ifdef USE_GOOGLE + InitGoogle(NULL, &argc, &argv, true); +#endif + run_tests(argc, argv); +} diff --git a/tests/upb_test.h b/tests/upb_test.h new file mode 100644 index 00000000000..d4b06883c3e --- /dev/null +++ b/tests/upb_test.h @@ -0,0 +1,53 @@ + +#ifndef UPB_TEST_H_ +#define UPB_TEST_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int num_assertions = 0; +uint32_t testhash = 0; + +#define PRINT_FAILURE(expr) \ + fprintf(stderr, "Assertion failed: %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, "expr: %s\n", #expr); \ + if (testhash) { \ + fprintf(stderr, "assertion failed running test %x. " \ + "Run with the arg %x to run only this test.\n", \ + testhash, testhash); \ + } + +#define ASSERT(expr) do { \ + ++num_assertions; \ + if (!(expr)) { \ + PRINT_FAILURE(expr) \ + abort(); \ + } \ +} while (0) + +#define ASSERT_NOCOUNT(expr) do { \ + if (!(expr)) { \ + PRINT_FAILURE(expr) \ + abort(); \ + } \ +} while (0) + +#define ASSERT_STATUS(expr, status) do { \ + ++num_assertions; \ + if (!(expr)) { \ + PRINT_FAILURE(expr) \ + fprintf(stderr, "failed status: %s\n", upb_status_errmsg(status)); \ + abort(); \ + } \ +} while (0) + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_DECODER_H_ */ diff --git a/third_party/lunit/LICENSE b/third_party/lunit/LICENSE new file mode 100644 index 00000000000..fb720fe4d64 --- /dev/null +++ b/third_party/lunit/LICENSE @@ -0,0 +1,32 @@ + +Lunit License +------------- + +Lunit is written by Michael Roth and is licensed +under the terms of the MIT license reproduced below. + +======================================================================== + +Copyright (c) 2004-2010 Michael Roth + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +======================================================================== + diff --git a/third_party/lunit/README.google b/third_party/lunit/README.google new file mode 100644 index 00000000000..af3e50e7396 --- /dev/null +++ b/third_party/lunit/README.google @@ -0,0 +1,9 @@ +URL: https://github.com/dcurrie/lunit +Version: 0.5 +License: MIT +License File: LICENSE +Description: +A unit testing library for Lua. + +Local Modifications: +Extracted the two file we actually need from the distribution. diff --git a/third_party/lunit/console.lua b/third_party/lunit/console.lua new file mode 100644 index 00000000000..0ff22a42967 --- /dev/null +++ b/third_party/lunit/console.lua @@ -0,0 +1,156 @@ + +--[[-------------------------------------------------------------------------- + + This file is part of lunit 0.5. + + For Details about lunit look at: http://www.mroth.net/lunit/ + + Author: Michael Roth + + Copyright (c) 2006-2008 Michael Roth + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--]]-------------------------------------------------------------------------- + + + +--[[ + + begin() + run(testcasename, testname) + err(fullname, message, traceback) + fail(fullname, where, message, usermessage) + pass(testcasename, testname) + done() + + Fullname: + testcase.testname + testcase.testname:setupname + testcase.testname:teardownname + +--]] + + +lunit = require "lunit" + +local lunit_console + +if _VERSION >= 'Lua 5.2' then + + lunit_console = setmetatable({},{__index = _ENV}) + _ENV = lunit_console + +else + + module( "lunit-console", package.seeall ) + lunit_console = _M + +end + + + +local function printformat(format, ...) + io.write( string.format(format, ...) ) +end + + +local columns_printed = 0 + +local function writestatus(char) + if columns_printed == 0 then + io.write(" ") + end + if columns_printed == 60 then + io.write("\n ") + columns_printed = 0 + end + io.write(char) + io.flush() + columns_printed = columns_printed + 1 +end + + +local msgs = {} + + +function begin() + local total_tc = 0 + local total_tests = 0 + + msgs = {} -- e + + for tcname in lunit.testcases() do + total_tc = total_tc + 1 + for testname, test in lunit.tests(tcname) do + total_tests = total_tests + 1 + end + end + + printformat("Loaded testsuite with %d tests in %d testcases.\n\n", total_tests, total_tc) +end + + +function run(testcasename, testname) + -- NOP +end + + +function err(fullname, message, traceback) + writestatus("E") + msgs[#msgs+1] = "Error! ("..fullname.."):\n"..message.."\n\t"..table.concat(traceback, "\n\t") .. "\n" +end + + +function fail(fullname, where, message, usermessage) + writestatus("F") + local text = "Failure ("..fullname.."):\n".. + where..": "..message.."\n" + + if usermessage then + text = text .. where..": "..usermessage.."\n" + end + + msgs[#msgs+1] = text +end + + +function pass(testcasename, testname) + writestatus(".") +end + + + +function done() + printformat("\n\n%d Assertions checked.\n", lunit.stats.assertions ) + print() + + for i, msg in ipairs(msgs) do + printformat( "%3d) %s\n", i, msg ) + end + + printformat("Testsuite finished (%d passed, %d failed, %d errors).\n", + lunit.stats.passed, lunit.stats.failed, lunit.stats.errors ) +end + + +return lunit_console + + diff --git a/third_party/lunit/lunit.lua b/third_party/lunit/lunit.lua new file mode 100644 index 00000000000..80f43c18e01 --- /dev/null +++ b/third_party/lunit/lunit.lua @@ -0,0 +1,725 @@ +--[[-------------------------------------------------------------------------- + + This file is part of lunit 0.5. + + For Details about lunit look at: http://www.mroth.net/lunit/ + + Author: Michael Roth + + Copyright (c) 2004, 2006-2010 Michael Roth + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--]]-------------------------------------------------------------------------- + + +local orig_assert = assert + +local pairs = pairs +local ipairs = ipairs +local next = next +local type = type +local error = error +local tostring = tostring +local setmetatable = setmetatable +local pcall = pcall +local xpcall = xpcall +local require = require +local loadfile = loadfile + +local string_sub = string.sub +local string_gsub = string.gsub +local string_format = string.format +local string_lower = string.lower +local string_find = string.find + +local table_concat = table.concat + +local debug_getinfo = debug.getinfo + +local _G = _G + +local lunit + +if _VERSION >= 'Lua 5.2' then + + lunit = {} + _ENV = lunit + +else + + module("lunit") + lunit = _M + +end + + +local __failure__ = {} -- Type tag for failed assertions + +local typenames = { "nil", "boolean", "number", "string", "table", "function", "thread", "userdata" } + + +local traceback_hide -- Traceback function which hides lunit internals +local mypcall -- Protected call to a function with own traceback +do + local _tb_hide = setmetatable( {}, {__mode="k"} ) + + function traceback_hide(func) + _tb_hide[func] = true + end + + local function my_traceback(errobj) + if is_table(errobj) and errobj.type == __failure__ then + local info = debug_getinfo(5, "Sl") -- FIXME: Hardcoded integers are bad... + errobj.where = string_format( "%s:%d", info.short_src, info.currentline) + else + errobj = { msg = tostring(errobj) } + errobj.tb = {} + local i = 2 + while true do + local info = debug_getinfo(i, "Snlf") + if not is_table(info) then + break + end + if not _tb_hide[info.func] then + local line = {} -- Ripped from ldblib.c... + line[#line+1] = string_format("%s:", info.short_src) + if info.currentline > 0 then + line[#line+1] = string_format("%d:", info.currentline) + end + if info.namewhat ~= "" then + line[#line+1] = string_format(" in function '%s'", info.name) + else + if info.what == "main" then + line[#line+1] = " in main chunk" + elseif info.what == "C" or info.what == "tail" then + line[#line+1] = " ?" + else + line[#line+1] = string_format(" in function <%s:%d>", info.short_src, info.linedefined) + end + end + errobj.tb[#errobj.tb+1] = table_concat(line) + end + i = i + 1 + end + end + return errobj + end + + function mypcall(func) + orig_assert( is_function(func) ) + local ok, errobj = xpcall(func, my_traceback) + if not ok then + return errobj + end + end + traceback_hide(mypcall) +end + + +-- Type check functions + +for _, typename in ipairs(typenames) do + lunit["is_"..typename] = function(x) + return type(x) == typename + end +end + +local is_nil = is_nil +local is_boolean = is_boolean +local is_number = is_number +local is_string = is_string +local is_table = is_table +local is_function = is_function +local is_thread = is_thread +local is_userdata = is_userdata + + +local function failure(name, usermsg, defaultmsg, ...) + local errobj = { + type = __failure__, + name = name, + msg = string_format(defaultmsg,...), + usermsg = usermsg + } + error(errobj, 0) +end +traceback_hide( failure ) + + +local function format_arg(arg) + local argtype = type(arg) + if argtype == "string" then + return "'"..arg.."'" + elseif argtype == "number" or argtype == "boolean" or argtype == "nil" then + return tostring(arg) + else + return "["..tostring(arg).."]" + end +end + + +local function selected(map, name) + if not map then + return true + end + + local m = {} + for k,v in pairs(map) do + m[k] = lunitpat2luapat(v) + end + return in_patternmap(m, name) +end + + +function fail(msg) + stats.assertions = stats.assertions + 1 + failure( "fail", msg, "failure" ) +end +traceback_hide( fail ) + + +function assert(assertion, msg) + stats.assertions = stats.assertions + 1 + if not assertion then + failure( "assert", msg, "assertion failed" ) + end + return assertion +end +traceback_hide( assert ) + + +function assert_true(actual, msg) + stats.assertions = stats.assertions + 1 + if actual ~= true then + failure( "assert_true", msg, "true expected but was %s", format_arg(actual) ) + end + return actual +end +traceback_hide( assert_true ) + + +function assert_false(actual, msg) + stats.assertions = stats.assertions + 1 + if actual ~= false then + failure( "assert_false", msg, "false expected but was %s", format_arg(actual) ) + end + return actual +end +traceback_hide( assert_false ) + + +function assert_equal(expected, actual, msg) + stats.assertions = stats.assertions + 1 + if expected ~= actual then + failure( "assert_equal", msg, "expected %s but was %s", format_arg(expected), format_arg(actual) ) + end + return actual +end +traceback_hide( assert_equal ) + + +function assert_not_equal(unexpected, actual, msg) + stats.assertions = stats.assertions + 1 + if unexpected == actual then + failure( "assert_not_equal", msg, "%s not expected but was one", format_arg(unexpected) ) + end + return actual +end +traceback_hide( assert_not_equal ) + + +function assert_match(pattern, actual, msg) + stats.assertions = stats.assertions + 1 + if type(pattern) ~= "string" then + failure( "assert_match", msg, "expected a string as pattern but was %s", format_arg(pattern) ) + end + if type(actual) ~= "string" then + failure( "assert_match", msg, "expected a string to match pattern '%s' but was a %s", pattern, format_arg(actual) ) + end + if not string_find(actual, pattern) then + failure( "assert_match", msg, "expected '%s' to match pattern '%s' but doesn't", actual, pattern ) + end + return actual +end +traceback_hide( assert_match ) + + +function assert_not_match(pattern, actual, msg) + stats.assertions = stats.assertions + 1 + if type(pattern) ~= "string" then + failure( "assert_not_match", msg, "expected a string as pattern but was %s", format_arg(pattern) ) + end + if type(actual) ~= "string" then + failure( "assert_not_match", msg, "expected a string to not match pattern '%s' but was %s", pattern, format_arg(actual) ) + end + if string_find(actual, pattern) then + failure( "assert_not_match", msg, "expected '%s' to not match pattern '%s' but it does", actual, pattern ) + end + return actual +end +traceback_hide( assert_not_match ) + + +function assert_error(msg, func) + stats.assertions = stats.assertions + 1 + if func == nil then + func, msg = msg, nil + end + if type(func) ~= "function" then + failure( "assert_error", msg, "expected a function as last argument but was %s", format_arg(func) ) + end + local ok, errmsg = pcall(func) + if ok then + failure( "assert_error", msg, "error expected but no error occurred" ) + end +end +traceback_hide( assert_error ) + + +function assert_error_match(msg, pattern, func) + stats.assertions = stats.assertions + 1 + if func == nil then + msg, pattern, func = nil, msg, pattern + end + if type(pattern) ~= "string" then + failure( "assert_error_match", msg, "expected the pattern as a string but was %s", format_arg(pattern) ) + end + if type(func) ~= "function" then + failure( "assert_error_match", msg, "expected a function as last argument but was %s", format_arg(func) ) + end + local ok, errmsg = pcall(func) + if ok then + failure( "assert_error_match", msg, "error expected but no error occurred" ) + end + if type(errmsg) ~= "string" then + failure( "assert_error_match", msg, "error as string expected but was %s", format_arg(errmsg) ) + end + if not string_find(errmsg, pattern) then + failure( "assert_error_match", msg, "expected error '%s' to match pattern '%s' but doesn't", errmsg, pattern ) + end +end +traceback_hide( assert_error_match ) + + +function assert_pass(msg, func) + stats.assertions = stats.assertions + 1 + if func == nil then + func, msg = msg, nil + end + if type(func) ~= "function" then + failure( "assert_pass", msg, "expected a function as last argument but was %s", format_arg(func) ) + end + local ok, errmsg = pcall(func) + if not ok then + failure( "assert_pass", msg, "no error expected but error was: '%s'", errmsg ) + end +end +traceback_hide( assert_pass ) + + +-- lunit.assert_typename functions + +for _, typename in ipairs(typenames) do + local assert_typename = "assert_"..typename + lunit[assert_typename] = function(actual, msg) + stats.assertions = stats.assertions + 1 + if type(actual) ~= typename then + failure( assert_typename, msg, "%s expected but was %s", typename, format_arg(actual) ) + end + return actual + end + traceback_hide( lunit[assert_typename] ) +end + + +-- lunit.assert_not_typename functions + +for _, typename in ipairs(typenames) do + local assert_not_typename = "assert_not_"..typename + lunit[assert_not_typename] = function(actual, msg) + stats.assertions = stats.assertions + 1 + if type(actual) == typename then + failure( assert_not_typename, msg, typename.." not expected but was one" ) + end + end + traceback_hide( lunit[assert_not_typename] ) +end + + +function lunit.clearstats() + stats = { + assertions = 0; + passed = 0; + failed = 0; + errors = 0; + } +end + + +local report, reporterrobj +do + local testrunner + + function lunit.setrunner(newrunner) + if not ( is_table(newrunner) or is_nil(newrunner) ) then + return error("lunit.setrunner: Invalid argument", 0) + end + local oldrunner = testrunner + testrunner = newrunner + return oldrunner + end + + function lunit.loadrunner(name) + if not is_string(name) then + return error("lunit.loadrunner: Invalid argument", 0) + end + local ok, runner = pcall( require, name ) + if not ok then + return error("lunit.loadrunner: Can't load test runner: "..runner, 0) + end + return setrunner(runner) + end + + function lunit.getrunner() + return testrunner + end + + function report(event, ...) + local f = testrunner and testrunner[event] + if is_function(f) then + pcall(f, ...) + end + end + + function reporterrobj(context, tcname, testname, errobj) + local fullname = tcname .. "." .. testname + if context == "setup" then + fullname = fullname .. ":" .. setupname(tcname, testname) + elseif context == "teardown" then + fullname = fullname .. ":" .. teardownname(tcname, testname) + end + if errobj.type == __failure__ then + stats.failed = stats.failed + 1 + report("fail", fullname, errobj.where, errobj.msg, errobj.usermsg) + else + stats.errors = stats.errors + 1 + report("err", fullname, errobj.msg, errobj.tb) + end + end +end + + + +local function key_iter(t, k) + return (next(t,k)) +end + + +local testcase +do + -- Array with all registered testcases + local _testcases = {} + + -- Marks a module as a testcase. + -- Applied over a module from module("xyz", lunit.testcase). + function lunit.testcase(m) + orig_assert( is_table(m) ) + --orig_assert( m._M == m ) + orig_assert( is_string(m._NAME) ) + --orig_assert( is_string(m._PACKAGE) ) + + -- Register the module as a testcase + _testcases[m._NAME] = m + + -- Import lunit, fail, assert* and is_* function to the module/testcase + m.lunit = lunit + m.fail = lunit.fail + for funcname, func in pairs(lunit) do + if "assert" == string_sub(funcname, 1, 6) or "is_" == string_sub(funcname, 1, 3) then + m[funcname] = func + end + end + end + + function lunit.module(name,seeall) + local m = {} + if seeall == "seeall" then + setmetatable(m, { __index = _G }) + end + m._NAME = name + lunit.testcase(m) + return m + end + + -- Iterator (testcasename) over all Testcases + function lunit.testcases() + -- Make a copy of testcases to prevent confusing the iterator when + -- new testcase are defined + local _testcases2 = {} + for k,v in pairs(_testcases) do + _testcases2[k] = true + end + return key_iter, _testcases2, nil + end + + function testcase(tcname) + return _testcases[tcname] + end +end + + +do + -- Finds a function in a testcase case insensitive + local function findfuncname(tcname, name) + for key, value in pairs(testcase(tcname)) do + if is_string(key) and is_function(value) and string_lower(key) == name then + return key + end + end + end + + function lunit.setupname(tcname) + return findfuncname(tcname, "setup") + end + + function lunit.teardownname(tcname) + return findfuncname(tcname, "teardown") + end + + -- Iterator over all test names in a testcase. + -- Have to collect the names first in case one of the test + -- functions creates a new global and throws off the iteration. + function lunit.tests(tcname) + local testnames = {} + for key, value in pairs(testcase(tcname)) do + if is_string(key) and is_function(value) then + local lfn = string_lower(key) + if string_sub(lfn, 1, 4) == "test" or string_sub(lfn, -4) == "test" then + testnames[key] = true + end + end + end + return key_iter, testnames, nil + end +end + + + + +function lunit.runtest(tcname, testname) + orig_assert( is_string(tcname) ) + orig_assert( is_string(testname) ) + + if (not getrunner()) then + loadrunner("console") + end + + local function callit(context, func) + if func then + local err = mypcall(func) + if err then + reporterrobj(context, tcname, testname, err) + return false + end + end + return true + end + traceback_hide(callit) + + report("run", tcname, testname) + + local tc = testcase(tcname) + local setup = tc[setupname(tcname)] + local test = tc[testname] + local teardown = tc[teardownname(tcname)] + + local setup_ok = callit( "setup", setup ) + local test_ok = setup_ok and callit( "test", test ) + local teardown_ok = setup_ok and callit( "teardown", teardown ) + + if setup_ok and test_ok and teardown_ok then + stats.passed = stats.passed + 1 + report("pass", tcname, testname) + end +end +traceback_hide(runtest) + + + +function lunit.run(testpatterns) + clearstats() + report("begin") + for testcasename in lunit.testcases() do + -- Run tests in the testcases + for testname in lunit.tests(testcasename) do + if selected(testpatterns, testname) then + runtest(testcasename, testname) + end + end + end + report("done") + return stats +end +traceback_hide(run) + + +function lunit.loadonly() + clearstats() + report("begin") + report("done") + return stats +end + + + + + + + + + +local lunitpat2luapat +do + local conv = { + ["^"] = "%^", + ["$"] = "%$", + ["("] = "%(", + [")"] = "%)", + ["%"] = "%%", + ["."] = "%.", + ["["] = "%[", + ["]"] = "%]", + ["+"] = "%+", + ["-"] = "%-", + ["?"] = ".", + ["*"] = ".*" + } + function lunitpat2luapat(str) + --return "^" .. string.gsub(str, "%W", conv) .. "$" + -- Above was very annoying, if I want to run all the tests having to do with + -- RSS, I want to be able to do "-t rss" not "-t \*rss\*". + return string_gsub(str, "%W", conv) + end +end + + + +local function in_patternmap(map, name) + if map[name] == true then + return true + else + for _, pat in ipairs(map) do + if string_find(name, pat) then + return true + end + end + end + return false +end + + + + + + + + +-- Called from 'lunit' shell script. + +function main(argv) + argv = argv or {} + + -- FIXME: Error handling and error messages aren't nice. + + local function checkarg(optname, arg) + if not is_string(arg) then + return error("lunit.main: option "..optname..": argument missing.", 0) + end + end + + local function loadtestcase(filename) + if not is_string(filename) then + return error("lunit.main: invalid argument") + end + local chunk, err = loadfile(filename) + if err then + return error(err) + else + chunk() + end + end + + local testpatterns = nil + local doloadonly = false + + local i = 0 + while i < #argv do + i = i + 1 + local arg = argv[i] + if arg == "--loadonly" then + doloadonly = true + elseif arg == "--runner" or arg == "-r" then + local optname = arg; i = i + 1; arg = argv[i] + checkarg(optname, arg) + loadrunner(arg) + elseif arg == "--test" or arg == "-t" then + local optname = arg; i = i + 1; arg = argv[i] + checkarg(optname, arg) + testpatterns = testpatterns or {} + testpatterns[#testpatterns+1] = arg + elseif arg == "--help" or arg == "-h" then + print[[ +lunit 0.5 +Copyright (c) 2004-2009 Michael Roth +This program comes WITHOUT WARRANTY OF ANY KIND. + +Usage: lua test [OPTIONS] [--] scripts + +Options: + + -r, --runner RUNNER Testrunner to use, defaults to 'lunit-console'. + -t, --test PATTERN Which tests to run, may contain * or ? wildcards. + --loadonly Only load the tests. + -h, --help Print this help screen. + +Please report bugs to . +]] + return + elseif arg == "--" then + while i < #argv do + i = i + 1; arg = argv[i] + loadtestcase(arg) + end + else + loadtestcase(arg) + end + end + + if doloadonly then + return loadonly() + else + return run(testpatterns) + end +end + +clearstats() + +return lunit diff --git a/tools/amalgamate.py b/tools/amalgamate.py new file mode 100755 index 00000000000..bc083b38962 --- /dev/null +++ b/tools/amalgamate.py @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import sys +import re +import os + +INCLUDE_RE = re.compile('^#include "([^"]*)"$') + +def parse_include(line): + match = INCLUDE_RE.match(line) + return match.groups()[0] if match else None + +class Amalgamator: + def __init__(self, output_path): + self.include_paths = ["."] + self.included = set(["upb/port_def.inc", "upb/port_undef.inc"]) + self.output_h = open(output_path + "upb.h", "w") + self.output_c = open(output_path + "upb.c", "w") + + self.output_c.write("/* Amalgamated source file */\n") + self.output_c.write('#include "upb.h"\n') + self.output_c.write(open("upb/port_def.inc").read()) + + self.output_h.write("/* Amalgamated source file */\n") + self.output_h.write('#include ') + self.output_h.write(open("upb/port_def.inc").read()) + + def add_include_path(self, path): + self.include_paths.append(path) + + def finish(self): + self.output_c.write(open("upb/port_undef.inc").read()) + self.output_h.write(open("upb/port_undef.inc").read()) + + def _process_file(self, infile_name, outfile): + file = None + for path in self.include_paths: + try: + full_path = os.path.join(path, infile_name) + file = open(full_path) + break + except IOError: + pass + if not file: + raise RuntimeError("Couldn't open file " + infile_name) + + for line in file: + include = parse_include(line) + if include is not None and (include.startswith("upb") or + include.startswith("google")): + if include not in self.included: + self.included.add(include) + self._add_header(include) + else: + outfile.write(line) + + def _add_header(self, filename): + self._process_file(filename, self.output_h) + + def add_src(self, filename): + self._process_file(filename, self.output_c) + +# ---- main ---- + +output_path = sys.argv[1] +amalgamator = Amalgamator(output_path) +files = [] + +for arg in sys.argv[2:]: + arg = arg.strip() + if arg.startswith("-I"): + amalgamator.add_include_path(arg[2:]) + elif arg.endswith(".h") or arg.endswith(".inc"): + pass + else: + files.append(arg) + +for filename in files: + amalgamator.add_src(filename) + +amalgamator.finish() diff --git a/tools/make_cmakelists.py b/tools/make_cmakelists.py new file mode 100755 index 00000000000..a4923c8da07 --- /dev/null +++ b/tools/make_cmakelists.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python + +"""TODO(haberman): DO NOT SUBMIT without one-line documentation for make_cmakelists. + +TODO(haberman): DO NOT SUBMIT without a detailed description of make_cmakelists. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import textwrap +import os + +def StripColons(deps): + return map(lambda x: x[1:], deps) + +def IsSourceFile(name): + return name.endswith(".c") or name.endswith(".cc") + +class BuildFileFunctions(object): + def __init__(self, converter): + self.converter = converter + + def _add_deps(self, kwargs, keyword=""): + if "deps" not in kwargs: + return + self.converter.toplevel += "target_link_libraries(%s%s\n %s)\n" % ( + kwargs["name"], + keyword, + "\n ".join(StripColons(kwargs["deps"])) + ) + + def load(self, *args): + pass + + def cc_library(self, **kwargs): + if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator": + return + files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) + found_files = [] + for file in files: + if os.path.isfile(file): + found_files.append(file) + elif os.path.isfile("generated_for_cmake/" + file): + found_files.append("generated_for_cmake/" + file) + else: + print("Warning: no such file: " + file) + + if list(filter(IsSourceFile, files)): + # Has sources, make this a normal library. + self.converter.toplevel += "add_library(%s\n %s)\n" % ( + kwargs["name"], + "\n ".join(found_files) + ) + self._add_deps(kwargs) + else: + # Header-only library, have to do a couple things differently. + # For some info, see: + # http://mariobadr.com/creating-a-header-only-library-with-cmake.html + self.converter.toplevel += "add_library(%s INTERFACE)\n" % ( + kwargs["name"] + ) + self._add_deps(kwargs, " INTERFACE") + + def cc_binary(self, **kwargs): + pass + + def cc_test(self, **kwargs): + # Disable this until we properly support upb_proto_library(). + # self.converter.toplevel += "add_executable(%s\n %s)\n" % ( + # kwargs["name"], + # "\n ".join(kwargs["srcs"]) + # ) + # self.converter.toplevel += "add_test(NAME %s COMMAND %s)\n" % ( + # kwargs["name"], + # kwargs["name"], + # ) + + # if "data" in kwargs: + # for data_dep in kwargs["data"]: + # self.converter.toplevel += textwrap.dedent("""\ + # add_custom_command( + # TARGET %s POST_BUILD + # COMMAND ${CMAKE_COMMAND} -E copy + # ${CMAKE_SOURCE_DIR}/%s + # ${CMAKE_CURRENT_BINARY_DIR}/%s)\n""" % ( + # kwargs["name"], data_dep, data_dep + # )) + + # self._add_deps(kwargs) + pass + + def py_library(self, **kwargs): + pass + + def py_binary(self, **kwargs): + pass + + def lua_cclibrary(self, **kwargs): + pass + + def lua_library(self, **kwargs): + pass + + def lua_binary(self, **kwargs): + pass + + def lua_test(self, **kwargs): + pass + + def sh_test(self, **kwargs): + pass + + def make_shell_script(self, **kwargs): + pass + + def exports_files(self, files, **kwargs): + pass + + def proto_library(self, **kwargs): + pass + + def generated_file_staleness_test(self, **kwargs): + pass + + def upb_amalgamation(self, **kwargs): + pass + + def upb_proto_library(self, **kwargs): + pass + + def upb_proto_reflection_library(self, **kwargs): + pass + + def upb_proto_srcs(self, **kwargs): + pass + + def genrule(self, **kwargs): + pass + + def config_setting(self, **kwargs): + pass + + def select(self, arg_dict): + return [] + + def glob(self, *args): + return [] + + def licenses(self, *args): + pass + + def filegroup(self, **kwargs): + pass + + def map_dep(self, arg): + return arg + + +class WorkspaceFileFunctions(object): + def __init__(self, converter): + self.converter = converter + + def load(self, *args): + pass + + def workspace(self, **kwargs): + self.converter.prelude += "project(%s)\n" % (kwargs["name"]) + + def http_archive(self, **kwargs): + pass + + def git_repository(self, **kwargs): + pass + + def bazel_version_repository(self, **kwargs): + pass + + def upb_deps(self): + pass + + +class Converter(object): + def __init__(self): + self.prelude = "" + self.toplevel = "" + self.if_lua = "" + + def convert(self): + return self.template % { + "prelude": converter.prelude, + "toplevel": converter.toplevel, + } + + template = textwrap.dedent("""\ + # This file was generated from BUILD using tools/make_cmakelists.py. + + cmake_minimum_required(VERSION 3.1) + + if(${CMAKE_VERSION} VERSION_LESS 3.12) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) + else() + cmake_policy(VERSION 3.12) + endif() + + cmake_minimum_required (VERSION 3.0) + cmake_policy(SET CMP0048 NEW) + + %(prelude)s + + # Prevent CMake from setting -rdynamic on Linux (!!). + SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") + SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + + # Set default build type. + if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) + endif() + + # When using Ninja, compiler output won't be colorized without this. + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS) + if(SUPPORTS_COLOR_ALWAYS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + endif() + + # Implement ASAN/UBSAN options + if(UPB_ENABLE_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + endif() + + if(UPB_ENABLE_UBSAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + endif() + + include_directories(.) + include_directories(generated_for_cmake) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) + + if(APPLE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") + elseif(UNIX) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") + endif() + + enable_testing() + + %(toplevel)s + + """) + +data = {} +converter = Converter() + +def GetDict(obj): + ret = {} + for k in dir(obj): + if not k.startswith("_"): + ret[k] = getattr(obj, k); + return ret + +globs = GetDict(converter) + +exec(open("WORKSPACE").read(), GetDict(WorkspaceFileFunctions(converter))) +exec(open("BUILD").read(), GetDict(BuildFileFunctions(converter))) + +with open(sys.argv[1], "w") as f: + f.write(converter.convert()) diff --git a/tools/staleness_test.py b/tools/staleness_test.py new file mode 100644 index 00000000000..045cd1a319d --- /dev/null +++ b/tools/staleness_test.py @@ -0,0 +1,30 @@ +"""The py_test() script for generated_file_staleness_test() rules. + +Note that this file is preprocessed! The INSERT_<...> text below is replaced +with the actual list of files before we actually run the script. +""" + +from __future__ import absolute_import + +from tools import staleness_test_lib +import unittest +import sys + +file_list = """ + INSERT_FILE_LIST_HERE +""".split() + +config = staleness_test_lib.Config(file_list) + + +class TestFilesMatch(unittest.TestCase): + + def testFilesMatch(self): + errors = staleness_test_lib.CheckFilesMatch(config) + self.assertFalse(errors, errors) + + +if len(sys.argv) > 1 and sys.argv[1] == "--fix": + staleness_test_lib.FixFiles(config) +else: + unittest.main() diff --git a/tools/staleness_test_lib.py b/tools/staleness_test_lib.py new file mode 100644 index 00000000000..6d5c3d30683 --- /dev/null +++ b/tools/staleness_test_lib.py @@ -0,0 +1,158 @@ +"""Shared code for validating generated_file_staleness_test() rules. + +This code is used by test scripts generated from +generated_file_staleness_test() rules. +""" + +from __future__ import absolute_import +from __future__ import print_function + +import os +from shutil import copyfile + + +class _FilePair(object): + """Represents a single (target, generated) file pair.""" + + def __init__(self, target, generated): + self.target = target + self.generated = generated + + +class Config(object): + """Represents the configuration for a single staleness test target.""" + + def __init__(self, file_list): + # Duplicate to avoid modifying our arguments. + file_list = list(file_list) + + # The file list contains a few other bits of information at the end. + # This is packed by the code in build_defs.bzl. + self.target_name = file_list.pop() + self.package_name = file_list.pop() + self.pattern = file_list.pop() + + self.file_list = file_list + + +def _GetFilePairs(config): + """Generates the list of file pairs. + + Args: + config: a Config object representing this target's config. + + Returns: + A list of _FilePair objects. + """ + + ret = [] + + has_bazel_genfiles = os.path.exists("bazel-genfiles") + + for filename in config.file_list: + target = os.path.join(config.package_name, filename) + generated = os.path.join(config.package_name, config.pattern % filename) + if has_bazel_genfiles: + generated = os.path.join("bazel-genfiles", generated) + + # Generated files should always exist. Blaze should guarantee this before + # we are run. + if not os.path.isfile(generated): + print("Generated file '%s' does not exist." % generated) + print("Please run this command to generate it:") + print(" bazel build %s:%s" % (config.package_name, config.target_name)) + ret.append(_FilePair(target, generated)) + + return ret + + +def _GetMissingAndStaleFiles(file_pairs): + """Generates lists of missing and stale files. + + Args: + file_pairs: a list of _FilePair objects. + + Returns: + missing_files: a list of _FilePair objects representing missing files. + These target files do not exist at all. + stale_files: a list of _FilePair objects representing stale files. + These target files exist but have stale contents. + """ + + missing_files = [] + stale_files = [] + + for pair in file_pairs: + if not os.path.isfile(pair.target): + missing_files.append(pair) + continue + + generated = open(pair.generated).read() + target = open(pair.target).read() + if generated != target: + stale_files.append(pair) + + return missing_files, stale_files + + +def _CopyFiles(file_pairs): + """Copies all generated files to the corresponding target file. + + The target files must be writable already. + + Args: + file_pairs: a list of _FilePair objects that we want to copy. + """ + + for pair in file_pairs: + target_dir = os.path.dirname(pair.target) + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + copyfile(pair.generated, pair.target) + + +def FixFiles(config): + """Implements the --fix option: overwrites missing or out-of-date files. + + Args: + config: the Config object for this test. + """ + + file_pairs = _GetFilePairs(config) + missing_files, stale_files = _GetMissingAndStaleFiles(file_pairs) + + _CopyFiles(stale_files + missing_files) + + +def CheckFilesMatch(config): + """Checks whether each target file matches the corresponding generated file. + + Args: + config: the Config object for this test. + + Returns: + None if everything matches, otherwise a string error message. + """ + + diff_errors = [] + + file_pairs = _GetFilePairs(config) + missing_files, stale_files = _GetMissingAndStaleFiles(file_pairs) + + for pair in missing_files: + diff_errors.append("File %s does not exist" % pair.target) + continue + + for pair in stale_files: + diff_errors.append("File %s is out of date" % pair.target) + + if diff_errors: + error_msg = "Files out of date!\n\n" + error_msg += "To fix run THIS command:\n" + error_msg += " bazel-bin/%s/%s --fix\n\n" % (config.package_name, + config.target_name) + error_msg += "Errors:\n" + error_msg += " " + "\n ".join(diff_errors) + return error_msg + else: + return None diff --git a/upb/bindings/README b/upb/bindings/README new file mode 100644 index 00000000000..3e176c943bd --- /dev/null +++ b/upb/bindings/README @@ -0,0 +1,5 @@ +This directory contains code that interfaces upb with external C/C++ +libraries. Right now this is: + + * upb/bindings/lua: + a Lua extension that exposes upb to Lua programs via the Lua C API. diff --git a/upb/bindings/lua/def.c b/upb/bindings/lua/def.c new file mode 100644 index 00000000000..a9ab9a89db0 --- /dev/null +++ b/upb/bindings/lua/def.c @@ -0,0 +1,766 @@ + +#include +#include +#include +#include +#include "lauxlib.h" +#include "upb/bindings/lua/upb.h" +#include "upb/def.h" + +#define LUPB_ENUMDEF "lupb.enumdef" +#define LUPB_FIELDDEF "lupb.fielddef" +#define LUPB_FILEDEF "lupb.filedef" +#define LUPB_MSGDEF "lupb.msgdef" +#define LUPB_ONEOFDEF "lupb.oneof" +#define LUPB_SYMTAB "lupb.symtab" +#define LUPB_OBJCACHE "lupb.objcache" + +#define CHK(pred) \ + do { \ + upb_status status; \ + upb_status_clear(&status); \ + pred; \ + lupb_checkstatus(L, &status); \ + } while (0) + +/* lupb_wrapper ***************************************************************/ + +/* Wrappers around upb objects. */ + +/* Checks type; if it matches, pulls the pointer out of the wrapper. */ +void *lupb_checkwrapper(lua_State *L, int narg, const char *type) { + void *ud = lua_touserdata(L, narg); + void *ret; + + if (!ud) { + luaL_typerror(L, narg, "upb wrapper"); + } + + memcpy(&ret, ud, sizeof(ret)); + if (!ret) { + luaL_error(L, "called into dead object"); + } + + luaL_checkudata(L, narg, type); + return ret; +} + +void lupb_pushwrapper(lua_State *L, const void *obj, const char *type) { + void *ud; + + if (obj == NULL) { + lua_pushnil(L); + return; + } + + /* Lookup our cache in the registry (we don't put our objects in the registry + * directly because we need our cache to be a weak table). */ + lua_getfield(L, LUA_REGISTRYINDEX, LUPB_OBJCACHE); + UPB_ASSERT(!lua_isnil(L, -1)); /* Should have been created by luaopen_upb. */ + lua_pushlightuserdata(L, (void*)obj); + lua_rawget(L, -2); + /* Stack is now: objcache, cached value. */ + + if (lua_isnil(L, -1)) { + /* Remove bad cached value and push new value. */ + lua_pop(L, 1); + ud = lua_newuserdata(L, sizeof(*ud)); + memcpy(ud, &obj, sizeof(*ud)); + + luaL_getmetatable(L, type); + /* Should have been created by luaopen_upb. */ + lupb_assert(L, !lua_isnil(L, -1)); + lua_setmetatable(L, -2); + + /* Set it in the cache. */ + lua_pushlightuserdata(L, (void*)obj); + lua_pushvalue(L, -2); + lua_rawset(L, -4); + } + + lua_insert(L, -2); + lua_pop(L, 1); +} + +void lupb_msgdef_pushwrapper(lua_State *L, const upb_msgdef *m); +void lupb_oneofdef_pushwrapper(lua_State *L, const upb_oneofdef *o); +static void lupb_enumdef_pushwrapper(lua_State *L, const upb_enumdef *e); + + +/* lupb_fielddef **************************************************************/ + +void lupb_fielddef_pushwrapper(lua_State *L, const upb_fielddef *f) { + lupb_pushwrapper(L, f, LUPB_FIELDDEF); +} + +const upb_fielddef *lupb_fielddef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_FIELDDEF); +} + +static int lupb_fielddef_containingoneof(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_oneofdef_pushwrapper(L, upb_fielddef_containingoneof(f)); + return 1; +} + +static int lupb_fielddef_containingtype(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_msgdef_pushwrapper(L, upb_fielddef_containingtype(f)); + return 1; +} + +static int lupb_fielddef_default(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: + lupb_pushint32(L, upb_fielddef_defaultint32(f)); break; + case UPB_TYPE_INT64: + lupb_pushint64(L, upb_fielddef_defaultint64(f)); break; + case UPB_TYPE_UINT32: + lupb_pushuint32(L, upb_fielddef_defaultuint32(f)); break; + case UPB_TYPE_UINT64: + lupb_pushuint64(L, upb_fielddef_defaultuint64(f)); break; + case UPB_TYPE_DOUBLE: + lua_pushnumber(L, upb_fielddef_defaultdouble(f)); break; + case UPB_TYPE_FLOAT: + lua_pushnumber(L, upb_fielddef_defaultfloat(f)); break; + case UPB_TYPE_BOOL: + lua_pushboolean(L, upb_fielddef_defaultbool(f)); break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + size_t len; + const char *data = upb_fielddef_defaultstr(f, &len); + lua_pushlstring(L, data, len); + break; + } + case UPB_TYPE_MESSAGE: + return luaL_error(L, "Message fields do not have explicit defaults."); + } + return 1; +} + +static int lupb_fielddef_descriptortype(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushnumber(L, upb_fielddef_descriptortype(f)); + return 1; +} + +static int lupb_fielddef_getsel(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + upb_selector_t sel; + if (upb_handlers_getselector(f, luaL_checknumber(L, 2), &sel)) { + lua_pushinteger(L, sel); + return 1; + } else { + return 0; + } +} + +static int lupb_fielddef_hassubdef(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_hassubdef(f)); + return 1; +} + +static int lupb_fielddef_index(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushinteger(L, upb_fielddef_index(f)); + return 1; +} + +static int lupb_fielddef_isextension(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_isextension(f)); + return 1; +} + +static int lupb_fielddef_label(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushinteger(L, upb_fielddef_label(f)); + return 1; +} + +static int lupb_fielddef_lazy(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_lazy(f)); + return 1; +} + +static int lupb_fielddef_name(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushstring(L, upb_fielddef_name(f)); + return 1; +} + +static int lupb_fielddef_number(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + int32_t num = upb_fielddef_number(f); + if (num) + lua_pushinteger(L, num); + else + lua_pushnil(L); + return 1; +} + +static int lupb_fielddef_packed(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lua_pushboolean(L, upb_fielddef_packed(f)); + return 1; +} + +static int lupb_fielddef_msgsubdef(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_msgdef_pushwrapper(L, upb_fielddef_msgsubdef(f)); + return 1; +} + +static int lupb_fielddef_enumsubdef(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + lupb_enumdef_pushwrapper(L, upb_fielddef_enumsubdef(f)); + return 1; +} + +static int lupb_fielddef_type(lua_State *L) { + const upb_fielddef *f = lupb_fielddef_check(L, 1); + if (upb_fielddef_typeisset(f)) + lua_pushinteger(L, upb_fielddef_type(f)); + else + lua_pushnil(L); + return 1; +} + +static const struct luaL_Reg lupb_fielddef_m[] = { + {"containing_oneof", lupb_fielddef_containingoneof}, + {"containing_type", lupb_fielddef_containingtype}, + {"default", lupb_fielddef_default}, + {"descriptor_type", lupb_fielddef_descriptortype}, + {"getsel", lupb_fielddef_getsel}, + {"has_subdef", lupb_fielddef_hassubdef}, + {"index", lupb_fielddef_index}, + {"is_extension", lupb_fielddef_isextension}, + {"label", lupb_fielddef_label}, + {"lazy", lupb_fielddef_lazy}, + {"name", lupb_fielddef_name}, + {"number", lupb_fielddef_number}, + {"packed", lupb_fielddef_packed}, + {"msgsubdef", lupb_fielddef_msgsubdef}, + {"enumsubdef", lupb_fielddef_enumsubdef}, + {"type", lupb_fielddef_type}, + {NULL, NULL} +}; + + +/* lupb_oneofdef **************************************************************/ + +void lupb_oneofdef_pushwrapper(lua_State *L, const upb_oneofdef *o) { + lupb_pushwrapper(L, o, LUPB_ONEOFDEF); +} + +const upb_oneofdef *lupb_oneofdef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_ONEOFDEF); +} + +static int lupb_oneofdef_containingtype(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + lupb_msgdef_pushwrapper(L, upb_oneofdef_containingtype(o)); + return 1; +} + +static int lupb_oneofdef_field(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + int type = lua_type(L, 2); + const upb_fielddef *f; + if (type == LUA_TNUMBER) { + f = upb_oneofdef_itof(o, lua_tointeger(L, 2)); + } else if (type == LUA_TSTRING) { + f = upb_oneofdef_ntofz(o, lua_tostring(L, 2)); + } else { + const char *msg = lua_pushfstring(L, "number or string expected, got %s", + luaL_typename(L, 2)); + return luaL_argerror(L, 2, msg); + } + + lupb_fielddef_pushwrapper(L, f); + return 1; +} + +static int lupb_oneofiter_next(lua_State *L) { + upb_oneof_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_oneof_done(i)) return 0; + lupb_fielddef_pushwrapper(L, upb_oneof_iter_field(i)); + upb_oneof_next(i); + return 1; +} + +static int lupb_oneofdef_fields(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + upb_oneof_iter *i = lua_newuserdata(L, sizeof(upb_oneof_iter)); + upb_oneof_begin(i, o); + /* Need to guarantee that the msgdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_oneofiter_next, 2); + return 1; +} + +static int lupb_oneofdef_len(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + lua_pushinteger(L, upb_oneofdef_numfields(o)); + return 1; +} + +static int lupb_oneofdef_name(lua_State *L) { + const upb_oneofdef *o = lupb_oneofdef_check(L, 1); + lua_pushstring(L, upb_oneofdef_name(o)); + return 1; +} + +static const struct luaL_Reg lupb_oneofdef_m[] = { + {"containing_type", lupb_oneofdef_containingtype}, + {"field", lupb_oneofdef_field}, + {"fields", lupb_oneofdef_fields}, + {"name", lupb_oneofdef_name}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_oneofdef_mm[] = { + {"__len", lupb_oneofdef_len}, + {NULL, NULL} +}; + + +/* lupb_msgdef ****************************************************************/ + +typedef struct { + const upb_msgdef *md; +} lupb_msgdef; + +void lupb_msgdef_pushwrapper(lua_State *L, const upb_msgdef *m) { + lupb_pushwrapper(L, m, LUPB_MSGDEF); +} + +const upb_msgdef *lupb_msgdef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_MSGDEF); +} + +static int lupb_msgdef_len(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + lua_pushinteger(L, upb_msgdef_numfields(m)); + return 1; +} + +static int lupb_msgdef_field(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + int type = lua_type(L, 2); + const upb_fielddef *f; + if (type == LUA_TNUMBER) { + f = upb_msgdef_itof(m, lua_tointeger(L, 2)); + } else if (type == LUA_TSTRING) { + f = upb_msgdef_ntofz(m, lua_tostring(L, 2)); + } else { + const char *msg = lua_pushfstring(L, "number or string expected, got %s", + luaL_typename(L, 2)); + return luaL_argerror(L, 2, msg); + } + + lupb_fielddef_pushwrapper(L, f); + return 1; +} + +static int lupb_msgdef_lookupname(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + const upb_fielddef *f; + const upb_oneofdef *o; + if (!upb_msgdef_lookupnamez(m, lua_tostring(L, 2), &f, &o)) { + lua_pushnil(L); + } else if (o) { + lupb_oneofdef_pushwrapper(L, o); + } else { + lupb_fielddef_pushwrapper(L, f); + } + return 1; +} + +static int lupb_msgfielditer_next(lua_State *L) { + upb_msg_field_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_msg_field_done(i)) return 0; + lupb_fielddef_pushwrapper(L, upb_msg_iter_field(i)); + upb_msg_field_next(i); + return 1; +} + +static int lupb_msgdef_fields(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + upb_msg_field_iter *i = lua_newuserdata(L, sizeof(upb_msg_field_iter)); + upb_msg_field_begin(i, m); + /* Need to guarantee that the msgdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_msgfielditer_next, 2); + return 1; +} + +static int lupb_msgoneofiter_next(lua_State *L) { + upb_msg_oneof_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_msg_oneof_done(i)) return 0; + lupb_oneofdef_pushwrapper(L, upb_msg_iter_oneof(i)); + upb_msg_oneof_next(i); + return 1; +} + +static int lupb_msgdef_oneofs(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + upb_msg_oneof_iter *i = lua_newuserdata(L, sizeof(upb_msg_oneof_iter)); + upb_msg_oneof_begin(i, m); + /* Need to guarantee that the msgdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_msgoneofiter_next, 2); + return 1; +} + +static int lupb_msgdef_mapentry(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + lua_pushboolean(L, upb_msgdef_mapentry(m)); + return 1; +} + +static int lupb_msgdef_syntax(lua_State *L) { + const upb_msgdef *m = lupb_msgdef_check(L, 1); + lua_pushinteger(L, upb_msgdef_syntax(m)); + return 1; +} + +static const struct luaL_Reg lupb_msgdef_mm[] = { + {"__len", lupb_msgdef_len}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_msgdef_m[] = { + {"field", lupb_msgdef_field}, + {"fields", lupb_msgdef_fields}, + {"lookup_name", lupb_msgdef_lookupname}, + {"oneofs", lupb_msgdef_oneofs}, + {"syntax", lupb_msgdef_syntax}, + {"_map_entry", lupb_msgdef_mapentry}, + {NULL, NULL} +}; + + +/* lupb_enumdef ***************************************************************/ + +const upb_enumdef *lupb_enumdef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_ENUMDEF); +} + +static void lupb_enumdef_pushwrapper(lua_State *L, const upb_enumdef *e) { + lupb_pushwrapper(L, e, LUPB_ENUMDEF); +} + +static int lupb_enumdef_len(lua_State *L) { + const upb_enumdef *e = lupb_enumdef_check(L, 1); + lua_pushinteger(L, upb_enumdef_numvals(e)); + return 1; +} + +static int lupb_enumdef_value(lua_State *L) { + const upb_enumdef *e = lupb_enumdef_check(L, 1); + int type = lua_type(L, 2); + if (type == LUA_TNUMBER) { + /* Pushes "nil" for a NULL pointer. */ + int32_t key = lupb_checkint32(L, 2); + lua_pushstring(L, upb_enumdef_iton(e, key)); + } else if (type == LUA_TSTRING) { + const char *key = lua_tostring(L, 2); + int32_t num; + if (upb_enumdef_ntoiz(e, key, &num)) { + lua_pushinteger(L, num); + } else { + lua_pushnil(L); + } + } else { + const char *msg = lua_pushfstring(L, "number or string expected, got %s", + luaL_typename(L, 2)); + return luaL_argerror(L, 2, msg); + } + return 1; +} + +static int lupb_enumiter_next(lua_State *L) { + upb_enum_iter *i = lua_touserdata(L, lua_upvalueindex(1)); + if (upb_enum_done(i)) return 0; + lua_pushstring(L, upb_enum_iter_name(i)); + lua_pushinteger(L, upb_enum_iter_number(i)); + upb_enum_next(i); + return 2; +} + +static int lupb_enumdef_values(lua_State *L) { + const upb_enumdef *e = lupb_enumdef_check(L, 1); + upb_enum_iter *i = lua_newuserdata(L, sizeof(upb_enum_iter)); + upb_enum_begin(i, e); + /* Need to guarantee that the enumdef outlives the iter. */ + lua_pushvalue(L, 1); + lua_pushcclosure(L, &lupb_enumiter_next, 2); + return 1; +} + +static const struct luaL_Reg lupb_enumdef_mm[] = { + {"__len", lupb_enumdef_len}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_enumdef_m[] = { + {"value", lupb_enumdef_value}, + {"values", lupb_enumdef_values}, + {NULL, NULL} +}; + + +/* lupb_filedef ***************************************************************/ + +void lupb_filedef_pushwrapper(lua_State *L, const upb_filedef *f) { + lupb_pushwrapper(L, f, LUPB_FILEDEF); +} + +const upb_filedef *lupb_filedef_check(lua_State *L, int narg) { + return lupb_checkwrapper(L, narg, LUPB_FILEDEF); +} + +static int lupb_filedef_dep(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + int index = luaL_checkint(L, 2); + lupb_filedef_pushwrapper(L, upb_filedef_dep(f, index)); + return 1; +} + +static int lupb_filedef_depcount(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_depcount(f)); + return 1; +} + +static int lupb_filedef_enum(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + int index = luaL_checkint(L, 2); + lupb_enumdef_pushwrapper(L, upb_filedef_enum(f, index)); + return 1; +} + +static int lupb_filedef_enumcount(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_enumcount(f)); + return 1; +} + +static int lupb_filedef_msg(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + int index = luaL_checkint(L, 2); + lupb_msgdef_pushwrapper(L, upb_filedef_msg(f, index)); + return 1; +} + +static int lupb_filedef_msgcount(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_msgcount(f)); + return 1; +} + +static int lupb_filedef_name(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushstring(L, upb_filedef_name(f)); + return 1; +} + +static int lupb_filedef_package(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushstring(L, upb_filedef_package(f)); + return 1; +} + +static int lupb_filedef_syntax(lua_State *L) { + const upb_filedef *f = lupb_filedef_check(L, 1); + lua_pushnumber(L, upb_filedef_syntax(f)); + return 1; +} + +static const struct luaL_Reg lupb_filedef_m[] = { + {"dep", lupb_filedef_dep}, + {"depcount", lupb_filedef_depcount}, + {"enum", lupb_filedef_enum}, + {"enumcount", lupb_filedef_enumcount}, + {"msg", lupb_filedef_msg}, + {"msgcount", lupb_filedef_msgcount}, + {"name", lupb_filedef_name}, + {"package", lupb_filedef_package}, + {"syntax", lupb_filedef_syntax}, + {NULL, NULL} +}; + + +/* lupb_symtab ****************************************************************/ + +typedef struct { + upb_symtab *symtab; +} lupb_symtab; + +upb_symtab *lupb_symtab_check(lua_State *L, int narg) { + lupb_symtab *lsymtab = luaL_checkudata(L, narg, LUPB_SYMTAB); + if (!lsymtab->symtab) { + luaL_error(L, "called into dead object"); + } + return lsymtab->symtab; +} + +static int lupb_symtab_new(lua_State *L) { + lupb_symtab *lsymtab = lua_newuserdata(L, sizeof(*lsymtab)); + lsymtab->symtab = upb_symtab_new(); + luaL_getmetatable(L, LUPB_SYMTAB); + lua_setmetatable(L, -2); + return 1; +} + +static int lupb_symtab_gc(lua_State *L) { + lupb_symtab *lsymtab = luaL_checkudata(L, 1, LUPB_SYMTAB); + upb_symtab_free(lsymtab->symtab); + lsymtab->symtab = NULL; + return 0; +} + +/* TODO(haberman): perhaps this should take a message object instead of a + * serialized string once we have a good story for vending compiled-in + * messages. */ +static int lupb_symtab_add(lua_State *L) { + upb_arena *arena; + size_t i, n, len; + const google_protobuf_FileDescriptorProto *const *files; + google_protobuf_FileDescriptorSet *set; + upb_symtab *s = lupb_symtab_check(L, 1); + const char *str = luaL_checklstring(L, 2, &len); + + lupb_arena_new(L); + arena = lupb_arena_check(L, -1); + + set = google_protobuf_FileDescriptorSet_parse(str, len, arena); + + if (!set) { + luaL_argerror(L, 2, "failed to parse descriptor"); + } + + files = google_protobuf_FileDescriptorSet_file(set, &n); + for (i = 0; i < n; i++) { + CHK(upb_symtab_addfile(s, files[i], &status)); + } + + return 0; +} + +static int lupb_symtab_lookupmsg(lua_State *L) { + const upb_symtab *s = lupb_symtab_check(L, 1); + const upb_msgdef *m = upb_symtab_lookupmsg(s, luaL_checkstring(L, 2)); + lupb_msgdef_pushwrapper(L, m); + return 1; +} + +static int lupb_symtab_lookupenum(lua_State *L) { + const upb_symtab *s = lupb_symtab_check(L, 1); + const upb_enumdef *e = upb_symtab_lookupenum(s, luaL_checkstring(L, 2)); + lupb_enumdef_pushwrapper(L, e); + return 1; +} + +static const struct luaL_Reg lupb_symtab_m[] = { + {"add", lupb_symtab_add}, + {"lookup_msg", lupb_symtab_lookupmsg}, + {"lookup_enum", lupb_symtab_lookupenum}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_symtab_mm[] = { + {"__gc", lupb_symtab_gc}, + {NULL, NULL} +}; + +/* lupb toplevel **************************************************************/ + +static void lupb_setfieldi(lua_State *L, const char *field, int i) { + lua_pushinteger(L, i); + lua_setfield(L, -2, field); +} + +static const struct luaL_Reg lupbdef_toplevel_m[] = { + {"SymbolTable", lupb_symtab_new}, + {NULL, NULL} +}; + +void lupb_def_registertypes(lua_State *L) { + lupb_setfuncs(L, lupbdef_toplevel_m); + + /* Refcounted types. */ + lupb_register_type(L, LUPB_ENUMDEF, lupb_enumdef_m, lupb_enumdef_mm); + lupb_register_type(L, LUPB_FIELDDEF, lupb_fielddef_m, NULL); + lupb_register_type(L, LUPB_FILEDEF, lupb_filedef_m, NULL); + lupb_register_type(L, LUPB_MSGDEF, lupb_msgdef_m, lupb_msgdef_mm); + lupb_register_type(L, LUPB_ONEOFDEF, lupb_oneofdef_m, lupb_oneofdef_mm); + lupb_register_type(L, LUPB_SYMTAB, lupb_symtab_m, lupb_symtab_mm); + + /* Create our object cache. */ + lua_newtable(L); + lua_createtable(L, 0, 1); /* Cache metatable. */ + lua_pushstring(L, "v"); /* Values are weak. */ + lua_setfield(L, -2, "__mode"); + lua_setmetatable(L, -2); + lua_setfield(L, LUA_REGISTRYINDEX, LUPB_OBJCACHE); + + /* Register constants. */ + lupb_setfieldi(L, "LABEL_OPTIONAL", UPB_LABEL_OPTIONAL); + lupb_setfieldi(L, "LABEL_REQUIRED", UPB_LABEL_REQUIRED); + lupb_setfieldi(L, "LABEL_REPEATED", UPB_LABEL_REPEATED); + + lupb_setfieldi(L, "TYPE_DOUBLE", UPB_TYPE_DOUBLE); + lupb_setfieldi(L, "TYPE_FLOAT", UPB_TYPE_FLOAT); + lupb_setfieldi(L, "TYPE_INT64", UPB_TYPE_INT64); + lupb_setfieldi(L, "TYPE_UINT64", UPB_TYPE_UINT64); + lupb_setfieldi(L, "TYPE_INT32", UPB_TYPE_INT32); + lupb_setfieldi(L, "TYPE_BOOL", UPB_TYPE_BOOL); + lupb_setfieldi(L, "TYPE_STRING", UPB_TYPE_STRING); + lupb_setfieldi(L, "TYPE_MESSAGE", UPB_TYPE_MESSAGE); + lupb_setfieldi(L, "TYPE_BYTES", UPB_TYPE_BYTES); + lupb_setfieldi(L, "TYPE_UINT32", UPB_TYPE_UINT32); + lupb_setfieldi(L, "TYPE_ENUM", UPB_TYPE_ENUM); + + lupb_setfieldi(L, "DESCRIPTOR_TYPE_DOUBLE", UPB_DESCRIPTOR_TYPE_DOUBLE); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_FLOAT", UPB_DESCRIPTOR_TYPE_FLOAT); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_INT64", UPB_DESCRIPTOR_TYPE_INT64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_UINT64", UPB_DESCRIPTOR_TYPE_UINT64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_INT32", UPB_DESCRIPTOR_TYPE_INT32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_FIXED64", UPB_DESCRIPTOR_TYPE_FIXED64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_FIXED32", UPB_DESCRIPTOR_TYPE_FIXED32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_BOOL", UPB_DESCRIPTOR_TYPE_BOOL); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_STRING", UPB_DESCRIPTOR_TYPE_STRING); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_GROUP", UPB_DESCRIPTOR_TYPE_GROUP); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_MESSAGE", UPB_DESCRIPTOR_TYPE_MESSAGE); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_BYTES", UPB_DESCRIPTOR_TYPE_BYTES); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_UINT32", UPB_DESCRIPTOR_TYPE_UINT32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_ENUM", UPB_DESCRIPTOR_TYPE_ENUM); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SFIXED32", UPB_DESCRIPTOR_TYPE_SFIXED32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SFIXED64", UPB_DESCRIPTOR_TYPE_SFIXED64); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SINT32", UPB_DESCRIPTOR_TYPE_SINT32); + lupb_setfieldi(L, "DESCRIPTOR_TYPE_SINT64", UPB_DESCRIPTOR_TYPE_SINT64); + + lupb_setfieldi(L, "HANDLER_INT32", UPB_HANDLER_INT32); + lupb_setfieldi(L, "HANDLER_INT64", UPB_HANDLER_INT64); + lupb_setfieldi(L, "HANDLER_UINT32", UPB_HANDLER_UINT32); + lupb_setfieldi(L, "HANDLER_UINT64", UPB_HANDLER_UINT64); + lupb_setfieldi(L, "HANDLER_FLOAT", UPB_HANDLER_FLOAT); + lupb_setfieldi(L, "HANDLER_DOUBLE", UPB_HANDLER_DOUBLE); + lupb_setfieldi(L, "HANDLER_BOOL", UPB_HANDLER_BOOL); + lupb_setfieldi(L, "HANDLER_STARTSTR", UPB_HANDLER_STARTSTR); + lupb_setfieldi(L, "HANDLER_STRING", UPB_HANDLER_STRING); + lupb_setfieldi(L, "HANDLER_ENDSTR", UPB_HANDLER_ENDSTR); + lupb_setfieldi(L, "HANDLER_STARTSUBMSG", UPB_HANDLER_STARTSUBMSG); + lupb_setfieldi(L, "HANDLER_ENDSUBMSG", UPB_HANDLER_ENDSUBMSG); + lupb_setfieldi(L, "HANDLER_STARTSEQ", UPB_HANDLER_STARTSEQ); + lupb_setfieldi(L, "HANDLER_ENDSEQ", UPB_HANDLER_ENDSEQ); + + lupb_setfieldi(L, "SYNTAX_PROTO2", UPB_SYNTAX_PROTO2); + lupb_setfieldi(L, "SYNTAX_PROTO3", UPB_SYNTAX_PROTO3); +} diff --git a/upb/bindings/lua/msg.c b/upb/bindings/lua/msg.c new file mode 100644 index 00000000000..5e769b2f0dd --- /dev/null +++ b/upb/bindings/lua/msg.c @@ -0,0 +1,1060 @@ +/* +** lupb_msg -- Message/Array/Map objects in Lua/C that wrap upb/msg.h +*/ + +#include +#include +#include +#include +#include + +#include "lauxlib.h" +#include "upb/bindings/lua/upb.h" +#include "upb/handlers.h" +#include "upb/legacy_msg_reflection.h" +#include "upb/msg.h" + +#include "upb/port_def.inc" + +/* + * Message/Array/Map objects can be constructed in one of two ways: + * + * 1. To point to existing msg/array/map data inside an arena. + * 2. To create and uniquely own some brand new data. + * + * Case (1) is for when we've parsed some data into an arena (which is faster + * than parsing directly into Lua objects) or when we're pointing at some + * read-only data (like custom options in a def). + * + * Case (2) is for when a user creates the object directly in Lua. + * + * We use the userval of container objects (Message/Array/Map) to store + * references to sub-objects (Strings/Messages/Arrays/Maps). But we need to + * keep the userval in sync with the underlying upb_msg/upb_array/upb_map. + * We populate the userval lazily from the underlying data. + * + * This means that no one may remove/replace any String/Message/Array/Map + * field/entry in the underlying upb_{msg,array,map} behind our back. It's ok + * for entries to be added or for primitives to be modified, but *replacing* + * sub-containers is not. + * + * Luckily parse/merge follow this rule. However clear does not, so it's not + * safe to clear behind our back. + */ + +#define LUPB_ARENA "lupb.arena" + +#define LUPB_MSGCLASS "lupb.msgclass" +#define LUPB_MSGFACTORY "lupb.msgfactory" + +#define LUPB_ARRAY "lupb.array" +#define LUPB_MAP "lupb.map" +#define LUPB_MSG "lupb.msg" +#define LUPB_STRING "lupb.string" + +static int lupb_msg_pushnew(lua_State *L, int narg); + +/* Lazily creates the uservalue if it doesn't exist. */ +static void lupb_getuservalue(lua_State *L, int index) { + lua_getuservalue(L, index); + if (lua_isnil(L, -1)) { + /* Lazily create and set userval. */ + lua_pop(L, 1); /* nil. */ + lua_pushvalue(L, index); /* userdata copy. */ + lua_newtable(L); + lua_setuservalue(L, -2); + lua_pop(L, 1); /* userdata copy. */ + lua_getuservalue(L, index); + } + assert(!lua_isnil(L, -1)); +} + +static void lupb_uservalseti(lua_State *L, int userdata, int index, int val) { + lupb_getuservalue(L, userdata); + lua_pushvalue(L, val); + lua_rawseti(L, -2, index); + lua_pop(L, 1); /* Uservalue. */ +} + +static void lupb_uservalgeti(lua_State *L, int userdata, int index) { + lupb_getuservalue(L, userdata); + lua_rawgeti(L, -1, index); + lua_insert(L, -2); + lua_pop(L, 1); /* Uservalue. */ +} + +/* Pushes a new userdata with the given metatable. */ +static void *lupb_newuserdata(lua_State *L, size_t size, const char *type) { + void *ret = lua_newuserdata(L, size); + + /* Set metatable. */ + luaL_getmetatable(L, type); + UPB_ASSERT(!lua_isnil(L, -1)); /* Should have been created by luaopen_upb. */ + lua_setmetatable(L, -2); + + /* We don't set a uservalue here -- we lazily create it later if necessary. */ + + return ret; +} + + +/* lupb_arena *****************************************************************/ + +/* lupb_arena only exists to wrap a upb_arena. It is never exposed to users; + * it is an internal memory management detail. Other objects refer to this + * object from their userdata to keep the arena-owned data alive. */ + +typedef struct { + upb_arena *arena; +} lupb_arena; + +upb_arena *lupb_arena_check(lua_State *L, int narg) { + lupb_arena *a = luaL_checkudata(L, narg, LUPB_ARENA); + return a ? a->arena : NULL; +} + +int lupb_arena_new(lua_State *L) { + lupb_arena *a = lupb_newuserdata(L, sizeof(lupb_arena), LUPB_ARENA); + + /* TODO(haberman): use Lua alloc func as block allocator? Would need to + * verify that all cases of upb_malloc in msg/table are longjmp-safe. */ + a->arena = upb_arena_new(); + + return 1; +} + +char lupb_arena_cache_key; + +/* Returns the global lupb_arena func that was created in our luaopen(). + * Callers can be guaranteed that it will be alive as long as |L| is. + * TODO(haberman): we shouldn't use a global arena! We should have + * one arena for a parse, or per independently-created message. */ +upb_arena *lupb_arena_get(lua_State *L) { + upb_arena *arena; + + lua_pushlightuserdata(L, &lupb_arena_cache_key); + lua_gettable(L, LUA_REGISTRYINDEX); + arena = lua_touserdata(L, -1); + UPB_ASSERT(arena); + lua_pop(L, 1); + + return arena; +} + +static void lupb_arena_initsingleton(lua_State *L) { + lua_pushlightuserdata(L, &lupb_arena_cache_key); + lupb_arena_new(L); + lua_settable(L, LUA_REGISTRYINDEX); +} + +static int lupb_arena_gc(lua_State *L) { + upb_arena *a = lupb_arena_check(L, 1); + upb_arena_free(a); + return 0; +} + +static const struct luaL_Reg lupb_arena_mm[] = { + {"__gc", lupb_arena_gc}, + {NULL, NULL} +}; + + +/* lupb_msgfactory ************************************************************/ + +/* Userval contains a map of: + * [1] -> SymbolTable (to keep GC-reachable) + * [const upb_msgdef*] -> [lupb_msgclass userdata] + */ + +#define LUPB_MSGFACTORY_SYMTAB 1 + +typedef struct lupb_msgfactory { + upb_msgfactory *factory; +} lupb_msgfactory; + +static int lupb_msgclass_pushnew(lua_State *L, int factory, + const upb_msgdef *md); + +/* lupb_msgfactory helpers. */ + +static lupb_msgfactory *lupb_msgfactory_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_MSGFACTORY); +} + +static void lupb_msgfactory_pushmsgclass(lua_State *L, int narg, + const upb_msgdef *md) { + lupb_getuservalue(L, narg); + lua_pushlightuserdata(L, (void*)md); + lua_rawget(L, -2); + + if (lua_isnil(L, -1)) { + lua_pop(L, 1); + /* TODO: verify md is in symtab? */ + lupb_msgclass_pushnew(L, narg, md); + + /* Set in userval. */ + lua_pushlightuserdata(L, (void*)md); + lua_pushvalue(L, -2); + lua_rawset(L, -4); + } +} + +static int lupb_msgfactory_gc(lua_State *L) { + lupb_msgfactory *lfactory = lupb_msgfactory_check(L, 1); + + if (lfactory->factory) { + upb_msgfactory_free(lfactory->factory); + lfactory->factory = NULL; + } + + return 0; +} + +/* lupb_msgfactory Public API. */ + +/** + * lupb_msgfactory_new() + * + * Handles: + * msgfactory = upb.MessageFactory(symtab) + * + * Creates a new, empty MessageFactory for the given SymbolTable. + * Message classes will be created on demand when the user calls + * msgfactory.get_message_class(). + */ +static int lupb_msgfactory_new(lua_State *L) { + const upb_symtab *symtab = lupb_symtab_check(L, 1); + + lupb_msgfactory *lmsgfactory = + lupb_newuserdata(L, sizeof(lupb_msgfactory), LUPB_MSGFACTORY); + lmsgfactory->factory = upb_msgfactory_new(symtab); + lupb_uservalseti(L, -1, LUPB_MSGFACTORY_SYMTAB, 1); + + return 1; +} + +/** + * lupb_msgfactory_getmsgclass() + * + * Handles: + * MessageClass = factory.get_message_class(message_name) + */ +static int lupb_msgfactory_getmsgclass(lua_State *L) { + lupb_msgfactory *lfactory = lupb_msgfactory_check(L, 1); + const upb_symtab *symtab = upb_msgfactory_symtab(lfactory->factory); + const upb_msgdef *m = upb_symtab_lookupmsg(symtab, luaL_checkstring(L, 2)); + + if (!m) { + luaL_error(L, "No such message type: %s\n", lua_tostring(L, 2)); + } + + lupb_msgfactory_pushmsgclass(L, 1, m); + + return 1; +} + +static const struct luaL_Reg lupb_msgfactory_m[] = { + {"get_message_class", lupb_msgfactory_getmsgclass}, + {NULL, NULL} +}; + +static const struct luaL_Reg lupb_msgfactory_mm[] = { + {"__gc", lupb_msgfactory_gc}, + {NULL, NULL} +}; + + +/* lupb_msgclass **************************************************************/ + +/* Userval contains a map of: + * [1] -> MessageFactory (to keep GC-reachable) + * [const upb_msgdef*] -> [lupb_msgclass userdata] + */ + +#define LUPB_MSGCLASS_FACTORY 1 + +struct lupb_msgclass { + const upb_msglayout *layout; + const upb_msgdef *msgdef; + const lupb_msgfactory *lfactory; +}; + +/* Type-checks for assigning to a message field. */ +static upb_msgval lupb_array_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f); +static upb_msgval lupb_map_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f); +static const lupb_msgclass *lupb_msg_getsubmsgclass(lua_State *L, int narg, + const upb_fielddef *f); +static const lupb_msgclass *lupb_msg_msgclassfor(lua_State *L, int narg, + const upb_msgdef *md); + +const lupb_msgclass *lupb_msgclass_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_MSGCLASS); +} + +const upb_msglayout *lupb_msgclass_getlayout(lua_State *L, int narg) { + return lupb_msgclass_check(L, narg)->layout; +} + +const upb_msgdef *lupb_msgclass_getmsgdef(const lupb_msgclass *lmsgclass) { + return lmsgclass->msgdef; +} + +upb_msgfactory *lupb_msgclass_getfactory(const lupb_msgclass *lmsgclass) { + return lmsgclass->lfactory->factory; +} + +/** + * lupb_msgclass_typecheck() + * + * Verifies that the expected msgclass matches the actual. If not, raises a Lua + * error. + */ +static void lupb_msgclass_typecheck(lua_State *L, const lupb_msgclass *expected, + const lupb_msgclass *actual) { + if (expected != actual) { + luaL_error(L, "Message had incorrect type, expected '%s', got '%s'", + upb_msgdef_fullname(expected->msgdef), + upb_msgdef_fullname(actual->msgdef)); + } +} + +static const lupb_msgclass *lupb_msgclass_msgclassfor(lua_State *L, int narg, + const upb_msgdef *md) { + lupb_uservalgeti(L, narg, LUPB_MSGCLASS_FACTORY); + lupb_msgfactory_pushmsgclass(L, -1, md); + return lupb_msgclass_check(L, -1); +} + +/** + * lupb_msgclass_getsubmsgclass() + * + * Given a MessageClass at index |narg| and the submessage field |f|, returns + * the message class for this field. + * + * Currently we do a hash table lookup for this. If we wanted we could try to + * optimize this by caching these pointers in our msgclass, in an array indexed + * by field index. We would still need to fall back to calling msgclassfor(), + * unless we wanted to eagerly create message classes for all submessages. But + * for big schemas that might be a lot of things to build, and we might end up + * not using most of them. */ +static const lupb_msgclass *lupb_msgclass_getsubmsgclass(lua_State *L, int narg, + const upb_fielddef *f) { + if (upb_fielddef_type(f) != UPB_TYPE_MESSAGE) { + return NULL; + } + + return lupb_msgclass_msgclassfor(L, narg, upb_fielddef_msgsubdef(f)); +} + +static int lupb_msgclass_pushnew(lua_State *L, int factory, + const upb_msgdef *md) { + const lupb_msgfactory *lfactory = lupb_msgfactory_check(L, factory); + lupb_msgclass *lmc = lupb_newuserdata(L, sizeof(*lmc), LUPB_MSGCLASS); + + lupb_uservalseti(L, -1, LUPB_MSGCLASS_FACTORY, factory); + lmc->layout = upb_msgfactory_getlayout(lfactory->factory, md); + lmc->lfactory = lfactory; + lmc->msgdef = md; + + return 1; +} + +/* MessageClass Public API. */ + +/** + * lupb_msgclass_call() + * + * Handles: + * msg = MessageClass() + * + * Creates a new message from the given MessageClass. + */ +static int lupb_msgclass_call(lua_State *L) { + lupb_msg_pushnew(L, 1); + return 1; +} + +static const struct luaL_Reg lupb_msgclass_mm[] = { + {"__call", lupb_msgclass_call}, + {NULL, NULL} +}; + + +/* upb <-> Lua type conversion ************************************************/ + +static bool lupb_istypewrapped(upb_fieldtype_t type) { + return type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES || + type == UPB_TYPE_MESSAGE; +} + +static upb_msgval lupb_tomsgval(lua_State *L, upb_fieldtype_t type, int narg, + const lupb_msgclass *lmsgclass) { + switch (type) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: + return upb_msgval_int32(lupb_checkint32(L, narg)); + case UPB_TYPE_INT64: + return upb_msgval_int64(lupb_checkint64(L, narg)); + case UPB_TYPE_UINT32: + return upb_msgval_uint32(lupb_checkuint32(L, narg)); + case UPB_TYPE_UINT64: + return upb_msgval_uint64(lupb_checkuint64(L, narg)); + case UPB_TYPE_DOUBLE: + return upb_msgval_double(lupb_checkdouble(L, narg)); + case UPB_TYPE_FLOAT: + return upb_msgval_float(lupb_checkfloat(L, narg)); + case UPB_TYPE_BOOL: + return upb_msgval_bool(lupb_checkbool(L, narg)); + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + size_t len; + const char *ptr = lupb_checkstring(L, narg, &len); + return upb_msgval_makestr(ptr, len); + } + case UPB_TYPE_MESSAGE: + UPB_ASSERT(lmsgclass); + return upb_msgval_msg(lupb_msg_checkmsg(L, narg, lmsgclass)); + } + UPB_UNREACHABLE(); +} + +static void lupb_pushmsgval(lua_State *L, upb_fieldtype_t type, + upb_msgval val) { + switch (type) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: + lupb_pushint32(L, upb_msgval_getint32(val)); + return; + case UPB_TYPE_INT64: + lupb_pushint64(L, upb_msgval_getint64(val)); + return; + case UPB_TYPE_UINT32: + lupb_pushuint32(L, upb_msgval_getuint32(val)); + return; + case UPB_TYPE_UINT64: + lupb_pushuint64(L, upb_msgval_getuint64(val)); + return; + case UPB_TYPE_DOUBLE: + lupb_pushdouble(L, upb_msgval_getdouble(val)); + return; + case UPB_TYPE_FLOAT: + lupb_pushfloat(L, upb_msgval_getfloat(val)); + return; + case UPB_TYPE_BOOL: + lua_pushboolean(L, upb_msgval_getbool(val)); + return; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + case UPB_TYPE_MESSAGE: + break; /* Shouldn't call this function. */ + } + UPB_UNREACHABLE(); +} + + +/* lupb_array *****************************************************************/ + +/* A strongly typed array. Implemented by wrapping upb_array. + * + * - we only allow integer indices. + * - all entries must have the correct type. + * - we do not allow "holes" in the array; you can only assign to an existing + * index or one past the end (which will grow the array by one). + * + * For string/submessage entries we keep in the userval: + * + * [number index] -> [lupb_string/lupb_msg userdata] + */ + +typedef struct { + /* Only needed for array of message. This wastes space in the non-message + * case but simplifies the code. Could optimize away if desired. */ + const lupb_msgclass *lmsgclass; + upb_array *arr; + upb_fieldtype_t type; +} lupb_array; + +#define ARRAY_MSGCLASS_INDEX 0 + +static lupb_array *lupb_array_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_ARRAY); +} + +/** + * lupb_array_typecheck() + * + * Verifies that the lupb_array object at index |narg| can be safely assigned + * to the field |f| of the lupb_msg object at index |msg|. If this is safe, + * returns a upb_msgval representing the array. Otherwise, throws a Lua error. + */ +static upb_msgval lupb_array_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f) { + lupb_array *larray = lupb_array_check(L, narg); + + if (upb_array_type(larray->arr) != upb_fielddef_type(f) || + lupb_msg_getsubmsgclass(L, msg, f) != larray->lmsgclass) { + luaL_error(L, "Array had incorrect type (expected: %d, got: %d)", + (int)upb_fielddef_type(f), (int)upb_array_type(larray->arr)); + } + + if (upb_array_type(larray->arr) == UPB_TYPE_MESSAGE) { + lupb_msgclass_typecheck(L, lupb_msg_getsubmsgclass(L, msg, f), + larray->lmsgclass); + } + + return upb_msgval_arr(larray->arr); +} + +/** + * lupb_array_checkindex() + * + * Checks the array index at Lua stack index |narg| to verify that it is an + * integer between 1 and |max|, inclusively. Also corrects it to be zero-based + * for C. + * + * We use "int" because of lua_rawseti/lua_rawgeti -- can re-evaluate if we want + * arrays bigger than 2^31. + */ +static int lupb_array_checkindex(lua_State *L, int narg, uint32_t max) { + uint32_t n = lupb_checkuint32(L, narg); + if (n == 0 || n > max || n > INT_MAX) { + luaL_error(L, "Invalid array index: expected between 1 and %d", (int)max); + } + return n - 1; /* Lua uses 1-based indexing. :( */ +} + +/* lupb_array Public API */ + +static int lupb_array_new(lua_State *L) { + lupb_array *larray; + upb_fieldtype_t type; + const lupb_msgclass *lmsgclass = NULL; + + if (lua_type(L, 1) == LUA_TNUMBER) { + type = lupb_checkfieldtype(L, 1); + } else { + type = UPB_TYPE_MESSAGE; + lmsgclass = lupb_msgclass_check(L, 1); + lupb_uservalseti(L, -1, ARRAY_MSGCLASS_INDEX, 1); /* GC-root lmsgclass. */ + } + + larray = lupb_newuserdata(L, sizeof(*larray), LUPB_ARRAY); + larray->type = type; + larray->lmsgclass = lmsgclass; + larray->arr = upb_array_new(lupb_arena_get(L)); + + return 1; +} + +static int lupb_array_newindex(lua_State *L) { + lupb_array *larray = lupb_array_check(L, 1); + upb_fieldtype_t type = upb_array_type(larray->arr); + uint32_t n = lupb_array_checkindex(L, 2, upb_array_size(larray->arr) + 1); + upb_msgval msgval = lupb_tomsgval(L, type, 3, larray->lmsgclass); + + upb_array_set(larray->arr, larray->type, n, msgval, lupb_arena_get(L)); + + if (lupb_istypewrapped(type)) { + lupb_uservalseti(L, 1, n, 3); + } + + return 0; /* 1 for chained assignments? */ +} + +static int lupb_array_index(lua_State *L) { + lupb_array *larray = lupb_array_check(L, 1); + upb_array *array = larray->arr; + uint32_t n = lupb_array_checkindex(L, 2, upb_array_size(array)); + upb_fieldtype_t type = upb_array_type(array); + + if (lupb_istypewrapped(type)) { + lupb_uservalgeti(L, 1, n); + } else { + lupb_pushmsgval(L, upb_array_type(array), + upb_array_get(array, larray->type, n)); + } + + return 1; +} + +static int lupb_array_len(lua_State *L) { + lupb_array *larray = lupb_array_check(L, 1); + lua_pushnumber(L, upb_array_size(larray->arr)); + return 1; +} + +static const struct luaL_Reg lupb_array_mm[] = { + {"__index", lupb_array_index}, + {"__len", lupb_array_len}, + {"__newindex", lupb_array_newindex}, + {NULL, NULL} +}; + + +/* lupb_map *******************************************************************/ + +/* A map object. Implemented by wrapping upb_map. + * + * When the value type is string/bytes/message, the userval consists of: + * + * [Lua number/string] -> [lupb_string/lupb_msg userdata] + * + * For other value types we don't use the userdata. + */ + +typedef struct { + const lupb_msgclass *value_lmsgclass; + upb_map *map; +} lupb_map; + +#define MAP_MSGCLASS_INDEX 0 + +/* lupb_map internal functions */ + +static lupb_map *lupb_map_check(lua_State *L, int narg) { + return luaL_checkudata(L, narg, LUPB_ARRAY); +} + +/** + * lupb_map_typecheck() + * + * Checks that the lupb_map at index |narg| can be safely assigned to the + * field |f| of the message at index |msg|. If so, returns a upb_msgval for + * this map. Otherwise, raises a Lua error. + */ +static upb_msgval lupb_map_typecheck(lua_State *L, int narg, int msg, + const upb_fielddef *f) { + lupb_map *lmap = lupb_map_check(L, narg); + upb_map *map = lmap->map; + const upb_msgdef *entry = upb_fielddef_msgsubdef(f); + const upb_fielddef *key_field = upb_msgdef_itof(entry, UPB_MAPENTRY_KEY); + const upb_fielddef *value_field = upb_msgdef_itof(entry, UPB_MAPENTRY_VALUE); + + UPB_ASSERT(entry && key_field && value_field); + + if (upb_map_keytype(map) != upb_fielddef_type(key_field)) { + luaL_error(L, "Map key type invalid"); + } + + if (upb_map_valuetype(map) != upb_fielddef_type(value_field)) { + luaL_error(L, "Map had incorrect value type (expected: %s, got: %s)", + upb_fielddef_type(value_field), upb_map_valuetype(map)); + } + + if (upb_map_valuetype(map) == UPB_TYPE_MESSAGE) { + lupb_msgclass_typecheck( + L, lupb_msg_msgclassfor(L, msg, upb_fielddef_msgsubdef(value_field)), + lmap->value_lmsgclass); + } + + return upb_msgval_map(map); +} + +/* lupb_map Public API */ + +/** + * lupb_map_new + * + * Handles: + * new_map = upb.Map(key_type, value_type) + */ +static int lupb_map_new(lua_State *L) { + lupb_map *lmap; + upb_fieldtype_t key_type = lupb_checkfieldtype(L, 1); + upb_fieldtype_t value_type; + const lupb_msgclass *value_lmsgclass = NULL; + + if (lua_type(L, 2) == LUA_TNUMBER) { + value_type = lupb_checkfieldtype(L, 2); + } else { + value_type = UPB_TYPE_MESSAGE; + } + + lmap = lupb_newuserdata(L, sizeof(*lmap), LUPB_MAP); + + if (value_type == UPB_TYPE_MESSAGE) { + value_lmsgclass = lupb_msgclass_check(L, 2); + lupb_uservalseti(L, -1, MAP_MSGCLASS_INDEX, 2); /* GC-root lmsgclass. */ + } + + lmap->value_lmsgclass = value_lmsgclass; + lmap->map = upb_map_new(key_type, value_type, lupb_arena_get(L)); + + return 1; +} + +/** + * lupb_map_index + * + * Handles: + * map[key] + */ +static int lupb_map_index(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + upb_map *map = lmap->map; + upb_fieldtype_t valtype = upb_map_valuetype(map); + /* We don't always use "key", but this call checks the key type. */ + upb_msgval key = lupb_tomsgval(L, upb_map_keytype(map), 2, NULL); + + if (lupb_istypewrapped(valtype)) { + /* Userval contains the full map, lookup there by key. */ + lupb_getuservalue(L, 1); + lua_pushvalue(L, 2); + lua_rawget(L, -2); + + if (lua_isnil(L, -1)) { + /* TODO: lazy read from upb_map */ + } + } else { + /* Lookup in upb_map. */ + upb_msgval val; + if (upb_map_get(map, key, &val)) { + lupb_pushmsgval(L, upb_map_valuetype(map), val); + } else { + lua_pushnil(L); + } + } + + return 1; +} + +/** + * lupb_map_len + * + * Handles: + * map_len = #map + */ +static int lupb_map_len(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + lua_pushnumber(L, upb_map_size(lmap->map)); + return 1; +} + +/** + * lupb_map_newindex + * + * Handles: + * map[key] = val + * map[key] = nil # to remove from map + */ +static int lupb_map_newindex(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + upb_map *map = lmap->map; + upb_msgval key = lupb_tomsgval(L, upb_map_keytype(map), 2, NULL); + + if (lua_isnil(L, 3)) { + /* Delete from map. */ + upb_map_del(map, key); + + if (lupb_istypewrapped(upb_map_valuetype(map))) { + /* Delete in userval. */ + lupb_getuservalue(L, 1); + lua_pushvalue(L, 2); + lua_pushnil(L); + lua_rawset(L, -3); + lua_pop(L, 1); + } + } else { + /* Set in map. */ + upb_msgval val = + lupb_tomsgval(L, upb_map_valuetype(map), 3, lmap->value_lmsgclass); + + upb_map_set(map, key, val, NULL); + + if (lupb_istypewrapped(upb_map_valuetype(map))) { + /* Set in userval. */ + lupb_getuservalue(L, 1); + lua_pushvalue(L, 2); + lua_pushvalue(L, 3); + lua_rawset(L, -3); + lua_pop(L, 1); + } + } + + return 0; +} + +/* upb_mapiter [[[ */ + +static int lupb_mapiter_next(lua_State *L) { + upb_mapiter *i = lua_touserdata(L, lua_upvalueindex(1)); + lupb_map *lmap = lupb_map_check(L, 1); + upb_map *map = lmap->map; + + if (upb_mapiter_done(i)) { + return 0; + } + + lupb_pushmsgval(L, upb_map_keytype(map), upb_mapiter_key(i)); + lupb_pushmsgval(L, upb_map_valuetype(map), upb_mapiter_value(i)); + upb_mapiter_next(i); + + return 2; +} + +static int lupb_map_pairs(lua_State *L) { + lupb_map *lmap = lupb_map_check(L, 1); + + if (lupb_istypewrapped(upb_map_keytype(lmap->map)) || + lupb_istypewrapped(upb_map_valuetype(lmap->map))) { + /* Complex key or value type. + * Sync upb_map to userval if necessary, then iterate over userval. */ + + /* TODO: Lua tables don't know how many entries they have, gah!. */ + return 1; + } else { + /* Simple key and value type, iterate over the upb_map directly. */ + upb_mapiter *i = lua_newuserdata(L, upb_mapiter_sizeof()); + + upb_mapiter_begin(i, lmap->map); + lua_pushvalue(L, 1); + + /* Upvalues are [upb_mapiter, lupb_map]. */ + lua_pushcclosure(L, &lupb_mapiter_next, 2); + + return 1; + } +} + +/* upb_mapiter ]]] */ + +static const struct luaL_Reg lupb_map_mm[] = { + {"__index", lupb_map_index}, + {"__len", lupb_map_len}, + {"__newindex", lupb_map_newindex}, + {"__pairs", lupb_map_pairs}, + {NULL, NULL} +}; + + +/* lupb_msg *******************************************************************/ + +/* A message object. Implemented by wrapping upb_msg. + * + * Our userval contains: + * + * - [0] -> our message class + * - [lupb_fieldindex(f)] -> [lupb_{string,array,map,msg} userdata] + * + * Fields with scalar number/bool types don't go in the userval. + */ + +#define LUPB_MSG_MSGCLASSINDEX 0 +#define LUPB_MSG_ARENA -1 + +int lupb_fieldindex(const upb_fielddef *f) { + return upb_fielddef_index(f) + 1; /* 1-based Lua arrays. */ +} + + +typedef struct { + const lupb_msgclass *lmsgclass; + upb_msg *msg; +} lupb_msg; + +/* lupb_msg helpers */ + +static bool in_userval(const upb_fielddef *f) { + return lupb_istypewrapped(upb_fielddef_type(f)) || upb_fielddef_isseq(f) || + upb_fielddef_ismap(f); +} + +lupb_msg *lupb_msg_check(lua_State *L, int narg) { + lupb_msg *msg = luaL_checkudata(L, narg, LUPB_MSG); + if (!msg->lmsgclass) luaL_error(L, "called into dead msg"); + return msg; +} + +const upb_msg *lupb_msg_checkmsg(lua_State *L, int narg, + const lupb_msgclass *lmsgclass) { + lupb_msg *lmsg = lupb_msg_check(L, narg); + lupb_msgclass_typecheck(L, lmsgclass, lmsg->lmsgclass); + return lmsg->msg; +} + +upb_msg *lupb_msg_checkmsg2(lua_State *L, int narg, + const upb_msglayout **layout) { + lupb_msg *lmsg = lupb_msg_check(L, narg); + *layout = lmsg->lmsgclass->layout; + return lmsg->msg; +} + +const upb_msgdef *lupb_msg_checkdef(lua_State *L, int narg) { + return lupb_msg_check(L, narg)->lmsgclass->msgdef; +} + +static const upb_fielddef *lupb_msg_checkfield(lua_State *L, + const lupb_msg *msg, + int fieldarg) { + size_t len; + const char *fieldname = luaL_checklstring(L, fieldarg, &len); + const upb_msgdef *msgdef = msg->lmsgclass->msgdef; + const upb_fielddef *f = upb_msgdef_ntof(msgdef, fieldname, len); + + if (!f) { + const char *msg = lua_pushfstring(L, "no such field: %s", fieldname); + luaL_argerror(L, fieldarg, msg); + return NULL; /* Never reached. */ + } + + return f; +} + +static const lupb_msgclass *lupb_msg_msgclassfor(lua_State *L, int narg, + const upb_msgdef *md) { + lupb_uservalgeti(L, narg, LUPB_MSG_MSGCLASSINDEX); + return lupb_msgclass_msgclassfor(L, -1, md); +} + +static const lupb_msgclass *lupb_msg_getsubmsgclass(lua_State *L, int narg, + const upb_fielddef *f) { + lupb_uservalgeti(L, narg, LUPB_MSG_MSGCLASSINDEX); + return lupb_msgclass_getsubmsgclass(L, -1, f); +} + +int lupb_msg_pushref(lua_State *L, int msgclass, upb_msg *msg) { + const lupb_msgclass *lmsgclass = lupb_msgclass_check(L, msgclass); + lupb_msg *lmsg = lupb_newuserdata(L, sizeof(lupb_msg), LUPB_MSG); + + lmsg->lmsgclass = lmsgclass; + lmsg->msg = msg; + + lupb_uservalseti(L, -1, LUPB_MSG_MSGCLASSINDEX, msgclass); + lupb_uservalseti(L, -1, LUPB_MSG_ARENA, -2); + + return 1; +} + +/* lupb_msg Public API */ + +/** + * lupb_msg_pushnew + * + * Handles: + * new_msg = MessageClass() + */ +static int lupb_msg_pushnew(lua_State *L, int narg) { + const lupb_msgclass *lmsgclass = lupb_msgclass_check(L, narg); + lupb_msg *lmsg = lupb_newuserdata(L, sizeof(lupb_msg), LUPB_MSG); + + lmsg->lmsgclass = lmsgclass; + lmsg->msg = upb_msg_new(lmsgclass->layout, lupb_arena_get(L)); + + lupb_uservalseti(L, -1, LUPB_MSG_MSGCLASSINDEX, narg); + + return 1; +} + +/** + * lupb_msg_index + * + * Handles: + * msg.foo + * msg["foo"] + * msg[field_descriptor] # (for extensions) (TODO) + */ +static int lupb_msg_index(lua_State *L) { + lupb_msg *lmsg = lupb_msg_check(L, 1); + const upb_fielddef *f = lupb_msg_checkfield(L, lmsg, 2); + const upb_msglayout *l = lmsg->lmsgclass->layout; + int field_index = upb_fielddef_index(f); + + if (in_userval(f)) { + lupb_uservalgeti(L, 1, lupb_fieldindex(f)); + + if (lua_isnil(L, -1)) { + /* Check if we need to lazily create wrapper. */ + if (upb_fielddef_isseq(f)) { + /* TODO(haberman) */ + } else if (upb_fielddef_issubmsg(f)) { + /* TODO(haberman) */ + } else { + UPB_ASSERT(upb_fielddef_isstring(f)); + if (upb_msg_has(lmsg->msg, field_index, l)) { + upb_msgval val = upb_msg_get(lmsg->msg, field_index, l); + lua_pop(L, 1); + lua_pushlstring(L, val.str.data, val.str.size); + lupb_uservalseti(L, 1, lupb_fieldindex(f), -1); + } + } + } + } else { + upb_msgval val = upb_msg_get(lmsg->msg, field_index, l); + lupb_pushmsgval(L, upb_fielddef_type(f), val); + } + + return 1; +} + +/** + * lupb_msg_newindex() + * + * Handles: + * msg.foo = bar + * msg["foo"] = bar + * msg[field_descriptor] = bar # (for extensions) (TODO) + */ +static int lupb_msg_newindex(lua_State *L) { + lupb_msg *lmsg = lupb_msg_check(L, 1); + const upb_fielddef *f = lupb_msg_checkfield(L, lmsg, 2); + upb_fieldtype_t type = upb_fielddef_type(f); + int field_index = upb_fielddef_index(f); + upb_msgval msgval; + + /* Typecheck and get msgval. */ + + if (upb_fielddef_isseq(f)) { + msgval = lupb_array_typecheck(L, 3, 1, f); + } else if (upb_fielddef_ismap(f)) { + msgval = lupb_map_typecheck(L, 3, 1, f); + } else { + const lupb_msgclass *lmsgclass = NULL; + + if (type == UPB_TYPE_MESSAGE) { + lmsgclass = lupb_msg_getsubmsgclass(L, 1, f); + } + + msgval = lupb_tomsgval(L, type, 3, lmsgclass); + } + + /* Set in upb_msg and userval (if necessary). */ + + upb_msg_set(lmsg->msg, field_index, msgval, lmsg->lmsgclass->layout); + + if (in_userval(f)) { + lupb_uservalseti(L, 1, lupb_fieldindex(f), 3); + } + + return 0; /* 1 for chained assignments? */ +} + +static const struct luaL_Reg lupb_msg_mm[] = { + {"__index", lupb_msg_index}, + {"__newindex", lupb_msg_newindex}, + {NULL, NULL} +}; + + +/* lupb_msg toplevel **********************************************************/ + +static const struct luaL_Reg lupb_msg_toplevel_m[] = { + {"Array", lupb_array_new}, + {"Map", lupb_map_new}, + {"MessageFactory", lupb_msgfactory_new}, + {NULL, NULL} +}; + +void lupb_msg_registertypes(lua_State *L) { + lupb_setfuncs(L, lupb_msg_toplevel_m); + + lupb_register_type(L, LUPB_ARENA, NULL, lupb_arena_mm); + lupb_register_type(L, LUPB_MSGCLASS, NULL, lupb_msgclass_mm); + lupb_register_type(L, LUPB_MSGFACTORY, lupb_msgfactory_m, lupb_msgfactory_mm); + lupb_register_type(L, LUPB_ARRAY, NULL, lupb_array_mm); + lupb_register_type(L, LUPB_MAP, NULL, lupb_map_mm); + lupb_register_type(L, LUPB_MSG, NULL, lupb_msg_mm); + + lupb_arena_initsingleton(L); +} diff --git a/upb/bindings/lua/upb.c b/upb/bindings/lua/upb.c new file mode 100644 index 00000000000..38fd24a27ef --- /dev/null +++ b/upb/bindings/lua/upb.c @@ -0,0 +1,245 @@ +/* +** require("lua") -- A Lua extension for upb. +** +** Exposes only the core library +** (sub-libraries are exposed in other extensions). +** +** 64-bit woes: Lua can only represent numbers of type lua_Number (which is +** double unless the user specifically overrides this). Doubles can represent +** the entire range of 64-bit integers, but lose precision once the integers are +** greater than 2^53. +** +** Lua 5.3 is adding support for integers, which will allow for 64-bit +** integers (which can be interpreted as signed or unsigned). +** +** LuaJIT supports 64-bit signed and unsigned boxed representations +** through its "cdata" mechanism, but this is not portable to regular Lua. +** +** Hopefully Lua 5.3 will come soon enough that we can either use Lua 5.3 +** integer support or LuaJIT 64-bit cdata for users that need the entire +** domain of [u]int64 values. +*/ + +#include +#include +#include +#include +#include "lauxlib.h" +#include "upb/bindings/lua/upb.h" +#include "upb/handlers.h" +#include "upb/msg.h" + + +/* Lua compatibility code *****************************************************/ + +/* Lua 5.1 and Lua 5.2 have slightly incompatible APIs. A little bit of + * compatibility code can help hide the difference. Not too many people still + * use Lua 5.1 but LuaJIT uses the Lua 5.1 API in some ways. */ + +#if LUA_VERSION_NUM == 501 + +/* taken from lua 5.2's source. */ +void *luaL_testudata(lua_State *L, int ud, const char *tname) { + void *p = lua_touserdata(L, ud); + if (p != NULL) { /* value is a userdata? */ + if (lua_getmetatable(L, ud)) { /* does it have a metatable? */ + luaL_getmetatable(L, tname); /* get correct metatable */ + if (!lua_rawequal(L, -1, -2)) /* not the same? */ + p = NULL; /* value is a userdata with wrong metatable */ + lua_pop(L, 2); /* remove both metatables */ + return p; + } + } + return NULL; /* value is not a userdata with a metatable */ +} + +static void lupb_newlib(lua_State *L, const char *name, const luaL_Reg *funcs) { + luaL_register(L, name, funcs); +} + +#elif LUA_VERSION_NUM == 502 + +int luaL_typerror(lua_State *L, int narg, const char *tname) { + const char *msg = lua_pushfstring(L, "%s expected, got %s", + tname, luaL_typename(L, narg)); + return luaL_argerror(L, narg, msg); +} + +static void lupb_newlib(lua_State *L, const char *name, const luaL_Reg *funcs) { + /* Lua 5.2 modules are not expected to set a global variable, so "name" is + * unused. */ + UPB_UNUSED(name); + + /* Can't use luaL_newlib(), because funcs is not the actual array. + * Could (micro-)optimize this a bit to count funcs for initial table size. */ + lua_createtable(L, 0, 8); + luaL_setfuncs(L, funcs, 0); +} + +#else +#error Only Lua 5.1 and 5.2 are supported +#endif + +/* Shims for upcoming Lua 5.3 functionality. */ +bool lua_isinteger(lua_State *L, int argn) { + UPB_UNUSED(L); + UPB_UNUSED(argn); + return false; +} + + +/* Utility functions **********************************************************/ + +/* We store our module table in the registry, keyed by ptr. + * For more info about the motivation/rationale, see this thread: + * http://thread.gmane.org/gmane.comp.lang.lua.general/110632 */ +bool lupb_openlib(lua_State *L, void *ptr, const char *name, + const luaL_Reg *funcs) { + /* Lookup cached module table. */ + lua_pushlightuserdata(L, ptr); + lua_rawget(L, LUA_REGISTRYINDEX); + if (!lua_isnil(L, -1)) { + return true; + } + + lupb_newlib(L, name, funcs); + + /* Save module table in cache. */ + lua_pushlightuserdata(L, ptr); + lua_pushvalue(L, -2); + lua_rawset(L, LUA_REGISTRYINDEX); + + return false; +} + +void lupb_checkstatus(lua_State *L, upb_status *s) { + if (!upb_ok(s)) { + lua_pushstring(L, upb_status_errmsg(s)); + lua_error(L); + } +} + +/* Scalar type mapping ********************************************************/ + +/* Functions that convert scalar/primitive values (numbers, strings, bool) + * between Lua and C/upb. Handles type/range checking. */ + +bool lupb_checkbool(lua_State *L, int narg) { + if (!lua_isboolean(L, narg)) { + luaL_error(L, "must be true or false"); + } + return lua_toboolean(L, narg); +} + +/* Unlike luaL_checkstring(), this does not allow implicit conversion to + * string. */ +const char *lupb_checkstring(lua_State *L, int narg, size_t *len) { + if (lua_type(L, narg) != LUA_TSTRING) { + luaL_error(L, "Expected string"); + } + + return lua_tolstring(L, narg, len); +} + +/* Unlike luaL_checkinteger, these do not implicitly convert from string or + * round an existing double value. We allow floating-point input, but only if + * the actual value is integral. */ +#define INTCHECK(type, ctype) \ + ctype lupb_check##type(lua_State *L, int narg) { \ + double n; \ + ctype i; \ + if (lua_isinteger(L, narg)) { \ + return lua_tointeger(L, narg); \ + } \ + \ + /* Prevent implicit conversion from string. */ \ + luaL_checktype(L, narg, LUA_TNUMBER); \ + n = lua_tonumber(L, narg); \ + \ + i = (ctype)n; \ + if ((double)i != n) { \ + /* double -> ctype truncated or rounded. */ \ + luaL_error(L, "number %f was not an integer or out of range for " #type, \ + n); \ + } \ + return i; \ + } \ + void lupb_push##type(lua_State *L, ctype val) { \ + /* TODO: push integer for Lua >= 5.3, 64-bit cdata for LuaJIT. */ \ + /* This is lossy for some [u]int64 values, which isn't great, but */ \ + /* crashing when we encounter these values seems worse. */ \ + lua_pushnumber(L, val); \ + } + +INTCHECK(int64, int64_t) +INTCHECK(int32, int32_t) +INTCHECK(uint64, uint64_t) +INTCHECK(uint32, uint32_t) + +double lupb_checkdouble(lua_State *L, int narg) { + /* If we were being really hard-nosed here, we'd check whether the input was + * an integer that has no precise double representation. But doubles aren't + * generally expected to be exact like integers are, and worse this could + * cause data-dependent runtime errors: one run of the program could work fine + * because the integer calculations happened to be exactly representable in + * double, while the next could crash because of subtly different input. */ + + luaL_checktype(L, narg, LUA_TNUMBER); /* lua_tonumber() auto-converts. */ + return lua_tonumber(L, narg); +} + +float lupb_checkfloat(lua_State *L, int narg) { + /* We don't worry about checking whether the input can be exactly converted to + * float -- see above. */ + + luaL_checktype(L, narg, LUA_TNUMBER); /* lua_tonumber() auto-converts. */ + return lua_tonumber(L, narg); +} + +void lupb_pushdouble(lua_State *L, double d) { + lua_pushnumber(L, d); +} + +void lupb_pushfloat(lua_State *L, float d) { + lua_pushnumber(L, d); +} + + +static const struct luaL_Reg lupb_toplevel_m[] = { + {NULL, NULL} +}; + +void lupb_register_type(lua_State *L, const char *name, const luaL_Reg *m, + const luaL_Reg *mm) { + luaL_newmetatable(L, name); + + if (mm) { + lupb_setfuncs(L, mm); + } + + if (m) { + /* Methods go in the mt's __index method. This implies that you can' + * implement __index and also have methods. */ + lua_getfield(L, -1, "__index"); + lupb_assert(L, lua_isnil(L, -1)); + lua_pop(L, 1); + + lua_createtable(L, 0, 0); + lupb_setfuncs(L, m); + lua_setfield(L, -2, "__index"); + } + + lua_pop(L, 1); /* The mt. */ +} + +int luaopen_upb_c(lua_State *L) { + static char module_key; + if (lupb_openlib(L, &module_key, "upb_c", lupb_toplevel_m)) { + return 1; + } + + lupb_def_registertypes(L); + lupb_msg_registertypes(L); + + return 1; /* Return package table. */ +} diff --git a/upb/bindings/lua/upb.h b/upb/bindings/lua/upb.h new file mode 100644 index 00000000000..51d8acf9e45 --- /dev/null +++ b/upb/bindings/lua/upb.h @@ -0,0 +1,127 @@ +/* +** Shared definitions for upb Lua modules. +*/ + +#ifndef UPB_LUA_UPB_H_ +#define UPB_LUA_UPB_H_ + +#include "lauxlib.h" +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/msg.h" +#include "upb/msgfactory.h" + +/* Lua 5.1/5.2 compatibility code. */ +#if LUA_VERSION_NUM == 501 + +#define lua_rawlen lua_objlen + +/* Lua >= 5.2's getuservalue/setuservalue functions do not exist in prior + * versions but the older function lua_getfenv() can provide 100% of its + * capabilities (the reverse is not true). */ +#define lua_getuservalue(L, index) lua_getfenv(L, index) +#define lua_setuservalue(L, index) lua_setfenv(L, index) + +void *luaL_testudata(lua_State *L, int ud, const char *tname); + +#define lupb_setfuncs(L, l) luaL_register(L, NULL, l) + +#elif LUA_VERSION_NUM == 502 + +int luaL_typerror(lua_State *L, int narg, const char *tname); + +#define lupb_setfuncs(L, l) luaL_setfuncs(L, l, 0) + +#else +#error Only Lua 5.1 and 5.2 are supported +#endif + +#define lupb_assert(L, predicate) \ + if (!(predicate)) \ + luaL_error(L, "internal error: %s, %s:%d ", #predicate, __FILE__, __LINE__); + +/* Function for initializing the core library. This function is idempotent, + * and should be called at least once before calling any of the functions that + * construct core upb types. */ +int luaopen_upb(lua_State *L); + +/* Gets or creates a package table for a C module that is uniquely identified by + * "ptr". The easiest way to supply a unique "ptr" is to pass the address of a + * static variable private in the module's .c file. + * + * If this module has already been registered in this lua_State, pushes it and + * returns true. + * + * Otherwise, creates a new module table for this module with the given name, + * pushes it, and registers the given top-level functions in it. It also sets + * it as a global variable, but only if the current version of Lua expects that + * (ie Lua 5.1/LuaJIT). + * + * If "false" is returned, the caller is guaranteed that this lib has not been + * registered in this Lua state before (regardless of any funny business the + * user might have done to the global state), so the caller can safely perform + * one-time initialization. */ +bool lupb_openlib(lua_State *L, void *ptr, const char *name, + const luaL_Reg *funcs); + +/* Custom check/push functions. Unlike the Lua equivalents, they are pinned to + * specific types (instead of lua_Number, etc), and do not allow any implicit + * conversion or data loss. */ +int64_t lupb_checkint64(lua_State *L, int narg); +int32_t lupb_checkint32(lua_State *L, int narg); +uint64_t lupb_checkuint64(lua_State *L, int narg); +uint32_t lupb_checkuint32(lua_State *L, int narg); +double lupb_checkdouble(lua_State *L, int narg); +float lupb_checkfloat(lua_State *L, int narg); +bool lupb_checkbool(lua_State *L, int narg); +const char *lupb_checkstring(lua_State *L, int narg, size_t *len); +const char *lupb_checkname(lua_State *L, int narg); + +void lupb_pushint64(lua_State *L, int64_t val); +void lupb_pushint32(lua_State *L, int32_t val); +void lupb_pushuint64(lua_State *L, uint64_t val); +void lupb_pushuint32(lua_State *L, uint32_t val); +void lupb_pushdouble(lua_State *L, double val); +void lupb_pushfloat(lua_State *L, float val); + +/* Registers a type with the given name, methods, and metamethods. */ +void lupb_register_type(lua_State *L, const char *name, const luaL_Reg *m, + const luaL_Reg *mm); + +/* Checks the given upb_status and throws a Lua error if it is not ok. */ +void lupb_checkstatus(lua_State *L, upb_status *s); + + +/** From def.c. ***************************************************************/ + +upb_fieldtype_t lupb_checkfieldtype(lua_State *L, int narg); + +const upb_msgdef *lupb_msgdef_check(lua_State *L, int narg); +const upb_enumdef *lupb_enumdef_check(lua_State *L, int narg); +const upb_fielddef *lupb_fielddef_check(lua_State *L, int narg); +upb_symtab *lupb_symtab_check(lua_State *L, int narg); + +void lupb_def_registertypes(lua_State *L); + + +/** From msg.c. ***************************************************************/ + +struct lupb_msgclass; +typedef struct lupb_msgclass lupb_msgclass; + +upb_arena *lupb_arena_check(lua_State *L, int narg); +int lupb_arena_new(lua_State *L); +upb_arena *lupb_arena_get(lua_State *L); +int lupb_msg_pushref(lua_State *L, int msgclass, void *msg); +const upb_msg *lupb_msg_checkmsg(lua_State *L, int narg, + const lupb_msgclass *lmsgclass); +upb_msg *lupb_msg_checkmsg2(lua_State *L, int narg, + const upb_msglayout **layout); + +const lupb_msgclass *lupb_msgclass_check(lua_State *L, int narg); +const upb_msglayout *lupb_msgclass_getlayout(lua_State *L, int narg); +const upb_msgdef *lupb_msgclass_getmsgdef(const lupb_msgclass *lmsgclass); +upb_msgfactory *lupb_msgclass_getfactory(const lupb_msgclass *lmsgclass); +void lupb_msg_registertypes(lua_State *L); + +#endif /* UPB_LUA_UPB_H_ */ diff --git a/upb/bindings/lua/upb.lua b/upb/bindings/lua/upb.lua new file mode 100644 index 00000000000..728852e997a --- /dev/null +++ b/upb/bindings/lua/upb.lua @@ -0,0 +1,172 @@ + +-- Before calling require on "upb_c", we need to load the same library +-- as RTLD_GLOBAL, for the benefit of other C extensions that depend on +-- C functions in the core. +-- +-- This has to happen *before* the require call, because if the module +-- is loaded RTLD_LOCAL first, a subsequent load as RTLD_GLOBAL won't +-- have the proper effect, at least on some platforms. +local so = package.searchpath and package.searchpath("upb_c", package.cpath) +if so then + package.loadlib(so, "*") +end + +local upb = require("upb_c") + +-- A convenience function for building/linking/freezing defs +-- while maintaining their original order. +-- +-- Sample usage: +-- local m1, m2 = upb.build_defs{ +-- upb.MessageDef{full_name = "M1", fields = { +-- upb.FieldDef{ +-- name = "m2", +-- number = 1, +-- type = upb.TYPE_MESSAGE, +-- subdef_name = ".M2" +-- }, +-- } +-- }, +-- upb.MessageDef{full_name = "M2"} +-- } +upb.build_defs = function(defs) + upb.SymbolTable(defs) + -- Lua 5.2 puts unpack in the table library. + return (unpack or table.unpack)(defs) +end + +local ipairs_iter = function(array, last_index) + local next_index = last_index + 1 + if next_index > #array then + return nil + end + return next_index, array[next_index] +end + +-- For iterating over the indexes and values of a upb.Array. +-- +-- for i, val in upb.ipairs(array) do +-- -- ... +-- end +upb.ipairs = function(array) + return ipairs_iter, array, 0 +end + +local set_named = function(obj, init) + for k, v in pairs(init) do + local func = obj["set_" .. k] + if not func then + error("Cannot set member: " .. k) + end + func(obj, v) + end +end + +-- Capture references to the functions we're wrapping. +local RealFieldDef = upb.FieldDef +local RealEnumDef = upb.EnumDef +local RealMessageDef = upb.MessageDef +local RealOneofDef = upb.OneofDef +local RealSymbolTable = upb.SymbolTable + +-- FieldDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5, +-- type=upb.TYPE_INT32, default_value=12, type_name="Foo"} +upb.FieldDef = function(init) + local f = RealFieldDef() + + if init then + -- Other members are often dependent on type, so set that first. + if init.type then + f:set_type(init.type) + init.type = nil + end + + set_named(f, init) + end + + return f +end + + +-- MessageDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}} +upb.MessageDef = function(init) + local m = RealMessageDef() + + if init then + for _, f in pairs(init.fields or {}) do + m:add(f) + end + init.fields = nil + + set_named(m, init) + end + + return m +end + +-- EnumDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.EnumDef{full_name="MyEnum", +-- values={ +-- {"FOO_VALUE_1", 1}, +-- {"FOO_VALUE_2", 2} +-- } +-- } +upb.EnumDef = function(init) + local e = RealEnumDef() + + if init then + for _, val in pairs(init.values or {}) do + e:add(val[1], val[2]) + end + init.values = nil + + set_named(e, init) + end + + return e +end + +-- OneofDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.OneofDef{name="foo", fields={...}} +upb.OneofDef = function(init) + local o = RealOneofDef() + + if init then + for _, val in pairs(init.fields or {}) do + o:add(val) + end + init.fields = nil + + set_named(o, init) + end + + return o +end + +-- SymbolTable constructor; a wrapper around the real constructor that can +-- add an initial set of defs. +upb.SymbolTable = function(defs) + local s = RealSymbolTable() + + if defs then + s:add(defs) + end + + return s +end + +return upb diff --git a/upb/bindings/lua/upb/pb.c b/upb/bindings/lua/upb/pb.c new file mode 100644 index 00000000000..266bd974c9e --- /dev/null +++ b/upb/bindings/lua/upb/pb.c @@ -0,0 +1,56 @@ +/* +** require("upb.pb") -- A Lua extension for upb.pb. +** +** Exposes all the types defined in upb/pb/{*}.h +** Also defines a few convenience functions on top. +*/ + +#include "upb/bindings/lua/upb.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#define LUPB_PBDECODERMETHOD "lupb.pb.decodermethod" + +static int lupb_pb_decode(lua_State *L) { + size_t len; + const upb_msglayout *layout; + upb_msg *msg = lupb_msg_checkmsg2(L, 1, &layout); + const char *pb = lua_tolstring(L, 2, &len); + + upb_decode(pb, len, msg, layout, lupb_arena_get(L)); + /* TODO(haberman): check for error. */ + + return 0; +} + +static int lupb_pb_encode(lua_State *L) { + const upb_msglayout *layout; + const upb_msg *msg = lupb_msg_checkmsg2(L, 1, &layout); + upb_arena *arena = upb_arena_new(); + size_t size; + char *result; + + result = upb_encode(msg, (const void*)layout, arena, &size); + + /* Free resources before we potentially bail on error. */ + lua_pushlstring(L, result, size); + upb_arena_free(arena); + /* TODO(haberman): check for error. */ + + return 1; +} + +static const struct luaL_Reg toplevel_m[] = { + {"decode", lupb_pb_decode}, + {"encode", lupb_pb_encode}, + {NULL, NULL} +}; + +int luaopen_upb_pb_c(lua_State *L) { + static char module_key; + if (lupb_openlib(L, &module_key, "upb.pb_c", toplevel_m)) { + return 1; + } + + return 1; +} diff --git a/upb/bindings/lua/upb/pb.lua b/upb/bindings/lua/upb/pb.lua new file mode 100644 index 00000000000..b865902d3a5 --- /dev/null +++ b/upb/bindings/lua/upb/pb.lua @@ -0,0 +1,3 @@ + +require "upb" +return require "upb.pb_c" diff --git a/upb/bindings/stdc++/string.h b/upb/bindings/stdc++/string.h new file mode 100644 index 00000000000..c3465481a34 --- /dev/null +++ b/upb/bindings/stdc++/string.h @@ -0,0 +1,69 @@ + +#ifndef UPB_STDCPP_H_ +#define UPB_STDCPP_H_ + +#include "upb/sink.h" + +#include "upb/port_def.inc" + +namespace upb { + +template +class FillStringHandler { + public: + static void SetHandler(upb_byteshandler* handler) { + upb_byteshandler_setstartstr(handler, &FillStringHandler::StartString, + NULL); + upb_byteshandler_setstring(handler, &FillStringHandler::StringBuf, NULL); + } + + private: + // TODO(haberman): add UpbBind/UpbMakeHandler support to BytesHandler so these + // can be prettier callbacks. + static void* StartString(void *c, const void *hd, size_t size) { + UPB_UNUSED(hd); + UPB_UNUSED(size); + + T* str = static_cast(c); + str->clear(); + return c; + } + + static size_t StringBuf(void* c, const void* hd, const char* buf, size_t n, + const upb_bufhandle* h) { + UPB_UNUSED(hd); + UPB_UNUSED(h); + + T* str = static_cast(c); + try { + str->append(buf, n); + return n; + } catch (const std::exception&) { + return 0; + } + } +}; + +class StringSink { + public: + template + explicit StringSink(T* target) { + // TODO(haberman): we need to avoid rebuilding a new handler every time, + // but with class globals disallowed for google3 C++ this is tricky. + upb_byteshandler_init(&handler_); + FillStringHandler::SetHandler(&handler_); + input_.Reset(&handler_, target); + } + + BytesSink input() { return input_; } + + private: + upb_byteshandler handler_; + BytesSink input_; +}; + +} // namespace upb + +#include "upb/port_undef.inc" + +#endif // UPB_STDCPP_H_ diff --git a/upb/decode.c b/upb/decode.c new file mode 100644 index 00000000000..88d4bb4fb6d --- /dev/null +++ b/upb/decode.c @@ -0,0 +1,604 @@ + +#include +#include "upb/upb.h" +#include "upb/decode.h" + +#include "upb/port_def.inc" + +/* Maps descriptor type -> upb field type. */ +const uint8_t upb_desctype_to_fieldtype[] = { + UPB_WIRE_TYPE_END_GROUP, /* ENDGROUP */ + UPB_TYPE_DOUBLE, /* DOUBLE */ + UPB_TYPE_FLOAT, /* FLOAT */ + UPB_TYPE_INT64, /* INT64 */ + UPB_TYPE_UINT64, /* UINT64 */ + UPB_TYPE_INT32, /* INT32 */ + UPB_TYPE_UINT64, /* FIXED64 */ + UPB_TYPE_UINT32, /* FIXED32 */ + UPB_TYPE_BOOL, /* BOOL */ + UPB_TYPE_STRING, /* STRING */ + UPB_TYPE_MESSAGE, /* GROUP */ + UPB_TYPE_MESSAGE, /* MESSAGE */ + UPB_TYPE_BYTES, /* BYTES */ + UPB_TYPE_UINT32, /* UINT32 */ + UPB_TYPE_ENUM, /* ENUM */ + UPB_TYPE_INT32, /* SFIXED32 */ + UPB_TYPE_INT64, /* SFIXED64 */ + UPB_TYPE_INT32, /* SINT32 */ + UPB_TYPE_INT64, /* SINT64 */ +}; + +/* Data pertaining to the parse. */ +typedef struct { + const char *ptr; /* Current parsing position. */ + const char *field_start; /* Start of this field. */ + const char *limit; /* End of delimited region or end of buffer. */ + upb_arena *arena; + int depth; + uint32_t end_group; /* Set to field number of END_GROUP tag, if any. */ +} upb_decstate; + +/* Data passed by value to each parsing function. */ +typedef struct { + char *msg; + const upb_msglayout *layout; + upb_decstate *state; +} upb_decframe; + +#define CHK(x) if (!(x)) { return 0; } + +static bool upb_skip_unknowngroup(upb_decstate *d, int field_number); +static bool upb_decode_message(upb_decstate *d, char *msg, + const upb_msglayout *l); + +static bool upb_decode_varint(const char **ptr, const char *limit, + uint64_t *val) { + uint8_t byte; + int bitpos = 0; + const char *p = *ptr; + *val = 0; + + do { + CHK(bitpos < 70 && p < limit); + byte = *p; + *val |= (uint64_t)(byte & 0x7F) << bitpos; + p++; + bitpos += 7; + } while (byte & 0x80); + + *ptr = p; + return true; +} + +static bool upb_decode_varint32(const char **ptr, const char *limit, + uint32_t *val) { + uint64_t u64; + CHK(upb_decode_varint(ptr, limit, &u64) && u64 <= UINT32_MAX); + *val = (uint32_t)u64; + return true; +} + +static bool upb_decode_64bit(const char **ptr, const char *limit, + uint64_t *val) { + CHK(limit - *ptr >= 8); + memcpy(val, *ptr, 8); + *ptr += 8; + return true; +} + +static bool upb_decode_32bit(const char **ptr, const char *limit, + uint32_t *val) { + CHK(limit - *ptr >= 4); + memcpy(val, *ptr, 4); + *ptr += 4; + return true; +} + +static int32_t upb_zzdecode_32(uint32_t n) { + return (n >> 1) ^ -(int32_t)(n & 1); +} + +static int64_t upb_zzdecode_64(uint64_t n) { + return (n >> 1) ^ -(int64_t)(n & 1); +} + +static bool upb_decode_string(const char **ptr, const char *limit, + int *outlen) { + uint32_t len; + + CHK(upb_decode_varint32(ptr, limit, &len) && + len < INT32_MAX && + limit - *ptr >= (int32_t)len); + + *outlen = len; + return true; +} + +static void upb_set32(void *msg, size_t ofs, uint32_t val) { + memcpy((char*)msg + ofs, &val, sizeof(val)); +} + +static bool upb_append_unknown(upb_decstate *d, upb_decframe *frame) { + upb_msg_addunknown(frame->msg, d->field_start, d->ptr - d->field_start, + d->arena); + return true; +} + + +static bool upb_skip_unknownfielddata(upb_decstate *d, uint32_t tag, + uint32_t group_fieldnum) { + switch (tag & 7) { + case UPB_WIRE_TYPE_VARINT: { + uint64_t val; + return upb_decode_varint(&d->ptr, d->limit, &val); + } + case UPB_WIRE_TYPE_32BIT: { + uint32_t val; + return upb_decode_32bit(&d->ptr, d->limit, &val); + } + case UPB_WIRE_TYPE_64BIT: { + uint64_t val; + return upb_decode_64bit(&d->ptr, d->limit, &val); + } + case UPB_WIRE_TYPE_DELIMITED: { + int len; + CHK(upb_decode_string(&d->ptr, d->limit, &len)); + d->ptr += len; + return true; + } + case UPB_WIRE_TYPE_START_GROUP: + return upb_skip_unknowngroup(d, tag >> 3); + case UPB_WIRE_TYPE_END_GROUP: + return (tag >> 3) == group_fieldnum; + } + return false; +} + +static bool upb_skip_unknowngroup(upb_decstate *d, int field_number) { + while (d->ptr < d->limit && d->end_group == 0) { + uint32_t tag = 0; + CHK(upb_decode_varint32(&d->ptr, d->limit, &tag)); + CHK(upb_skip_unknownfielddata(d, tag, field_number)); + } + + CHK(d->end_group == field_number); + d->end_group = 0; + return true; +} + +static bool upb_array_grow(upb_array *arr, size_t elements, size_t elem_size, + upb_arena *arena) { + size_t needed = arr->len + elements; + size_t new_size = UPB_MAX(arr->size, 8); + size_t new_bytes; + size_t old_bytes; + void *new_data; + upb_alloc *alloc = upb_arena_alloc(arena); + + while (new_size < needed) { + new_size *= 2; + } + + old_bytes = arr->len * elem_size; + new_bytes = new_size * elem_size; + new_data = upb_realloc(alloc, arr->data, old_bytes, new_bytes); + CHK(new_data); + + arr->data = new_data; + arr->size = new_size; + return true; +} + +static void *upb_array_reserve(upb_array *arr, size_t elements, + size_t elem_size, upb_arena *arena) { + if (arr->size - arr->len < elements) { + CHK(upb_array_grow(arr, elements, elem_size, arena)); + } + return (char*)arr->data + (arr->len * elem_size); +} + +bool upb_array_add(upb_array *arr, size_t elements, size_t elem_size, + const void *data, upb_arena *arena) { + void *dest = upb_array_reserve(arr, elements, elem_size, arena); + + CHK(dest); + arr->len += elements; + memcpy(dest, data, elements * elem_size); + + return true; +} + +static upb_array *upb_getarr(upb_decframe *frame, + const upb_msglayout_field *field) { + UPB_ASSERT(field->label == UPB_LABEL_REPEATED); + return *(upb_array**)&frame->msg[field->offset]; +} + +static upb_array *upb_getorcreatearr(upb_decframe *frame, + const upb_msglayout_field *field) { + upb_array *arr = upb_getarr(frame, field); + + if (!arr) { + arr = upb_array_new(frame->state->arena); + CHK(arr); + *(upb_array**)&frame->msg[field->offset] = arr; + } + + return arr; +} + +static upb_msg *upb_getorcreatemsg(upb_decframe *frame, + const upb_msglayout_field *field, + const upb_msglayout **subm) { + upb_msg **submsg = (void*)(frame->msg + field->offset); + *subm = frame->layout->submsgs[field->submsg_index]; + + UPB_ASSERT(field->label != UPB_LABEL_REPEATED); + + if (!*submsg) { + *submsg = upb_msg_new(*subm, frame->state->arena); + CHK(*submsg); + } + + return *submsg; +} + +static upb_msg *upb_addmsg(upb_decframe *frame, + const upb_msglayout_field *field, + const upb_msglayout **subm) { + upb_msg *submsg; + upb_array *arr = upb_getorcreatearr(frame, field); + + *subm = frame->layout->submsgs[field->submsg_index]; + submsg = upb_msg_new(*subm, frame->state->arena); + CHK(submsg); + upb_array_add(arr, 1, sizeof(submsg), &submsg, frame->state->arena); + + return submsg; +} + +static void upb_sethasbit(upb_decframe *frame, + const upb_msglayout_field *field) { + int32_t hasbit = field->presence; + UPB_ASSERT(field->presence > 0); + frame->msg[hasbit / 8] |= (1 << (hasbit % 8)); +} + +static void upb_setoneofcase(upb_decframe *frame, + const upb_msglayout_field *field) { + UPB_ASSERT(field->presence < 0); + upb_set32(frame->msg, ~field->presence, field->number); +} + +static bool upb_decode_addval(upb_decframe *frame, + const upb_msglayout_field *field, void *val, + size_t size) { + char *field_mem = frame->msg + field->offset; + upb_array *arr; + + if (field->label == UPB_LABEL_REPEATED) { + arr = upb_getorcreatearr(frame, field); + CHK(arr); + field_mem = upb_array_reserve(arr, 1, size, frame->state->arena); + CHK(field_mem); + } + + memcpy(field_mem, val, size); + return true; +} + +static void upb_decode_setpresent(upb_decframe *frame, + const upb_msglayout_field *field) { + if (field->label == UPB_LABEL_REPEATED) { + upb_array *arr = upb_getarr(frame, field); + UPB_ASSERT(arr->len < arr->size); + arr->len++; + } else if (field->presence < 0) { + upb_setoneofcase(frame, field); + } else if (field->presence > 0) { + upb_sethasbit(frame, field); + } +} + +static bool upb_decode_msgfield(upb_decstate *d, upb_msg *msg, + const upb_msglayout *layout, int limit) { + const char* saved_limit = d->limit; + d->limit = d->ptr + limit; + CHK(--d->depth >= 0); + upb_decode_message(d, msg, layout); + d->depth++; + d->limit = saved_limit; + CHK(d->end_group == 0); + return true; +} + +static bool upb_decode_groupfield(upb_decstate *d, upb_msg *msg, + const upb_msglayout *layout, + int field_number) { + CHK(--d->depth >= 0); + upb_decode_message(d, msg, layout); + d->depth++; + CHK(d->end_group == field_number); + d->end_group = 0; + return true; +} + +static bool upb_decode_varintfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + uint64_t val; + CHK(upb_decode_varint(&d->ptr, d->limit, &val)); + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + CHK(upb_decode_addval(frame, field, &val, sizeof(val))); + break; + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_UINT32: + case UPB_DESCRIPTOR_TYPE_ENUM: { + uint32_t val32 = (uint32_t)val; + CHK(upb_decode_addval(frame, field, &val32, sizeof(val32))); + break; + } + case UPB_DESCRIPTOR_TYPE_BOOL: { + bool valbool = val != 0; + CHK(upb_decode_addval(frame, field, &valbool, sizeof(valbool))); + break; + } + case UPB_DESCRIPTOR_TYPE_SINT32: { + int32_t decoded = upb_zzdecode_32((uint32_t)val); + CHK(upb_decode_addval(frame, field, &decoded, sizeof(decoded))); + break; + } + case UPB_DESCRIPTOR_TYPE_SINT64: { + int64_t decoded = upb_zzdecode_64(val); + CHK(upb_decode_addval(frame, field, &decoded, sizeof(decoded))); + break; + } + default: + return upb_append_unknown(d, frame); + } + + upb_decode_setpresent(frame, field); + return true; +} + +static bool upb_decode_64bitfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + uint64_t val; + CHK(upb_decode_64bit(&d->ptr, d->limit, &val)); + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + case UPB_DESCRIPTOR_TYPE_FIXED64: + case UPB_DESCRIPTOR_TYPE_SFIXED64: + CHK(upb_decode_addval(frame, field, &val, sizeof(val))); + break; + default: + return upb_append_unknown(d, frame); + } + + upb_decode_setpresent(frame, field); + return true; +} + +static bool upb_decode_32bitfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + uint32_t val; + CHK(upb_decode_32bit(&d->ptr, d->limit, &val)); + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_FLOAT: + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + CHK(upb_decode_addval(frame, field, &val, sizeof(val))); + break; + default: + return upb_append_unknown(d, frame); + } + + upb_decode_setpresent(frame, field); + return true; +} + +static bool upb_decode_fixedpacked(upb_decstate *d, upb_array *arr, + uint32_t len, int elem_size) { + size_t elements = len / elem_size; + + CHK((size_t)(elements * elem_size) == len); + CHK(upb_array_add(arr, elements, elem_size, d->ptr, d->arena)); + d->ptr += len; + + return true; +} + +static upb_strview upb_decode_strfield(upb_decstate *d, uint32_t len) { + upb_strview ret; + ret.data = d->ptr; + ret.size = len; + d->ptr += len; + return ret; +} + +static bool upb_decode_toarray(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field, int len) { + upb_array *arr = upb_getorcreatearr(frame, field); + CHK(arr); + +#define VARINT_CASE(ctype, decode) \ + VARINT_CASE_EX(ctype, decode, decode) + +#define VARINT_CASE_EX(ctype, decode, dtype) \ + { \ + const char *ptr = d->ptr; \ + const char *limit = ptr + len; \ + while (ptr < limit) { \ + uint64_t val; \ + ctype decoded; \ + CHK(upb_decode_varint(&ptr, limit, &val)); \ + decoded = (decode)((dtype)val); \ + CHK(upb_array_add(arr, 1, sizeof(decoded), &decoded, d->arena)); \ + } \ + d->ptr = ptr; \ + return true; \ + } + + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview str = upb_decode_strfield(d, len); + return upb_array_add(arr, 1, sizeof(str), &str, d->arena); + } + case UPB_DESCRIPTOR_TYPE_FLOAT: + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + return upb_decode_fixedpacked(d, arr, len, sizeof(int32_t)); + case UPB_DESCRIPTOR_TYPE_DOUBLE: + case UPB_DESCRIPTOR_TYPE_FIXED64: + case UPB_DESCRIPTOR_TYPE_SFIXED64: + return upb_decode_fixedpacked(d, arr, len, sizeof(int64_t)); + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_UINT32: + case UPB_DESCRIPTOR_TYPE_ENUM: + VARINT_CASE(uint32_t, uint32_t); + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + VARINT_CASE(uint64_t, uint64_t); + case UPB_DESCRIPTOR_TYPE_BOOL: + VARINT_CASE(bool, bool); + case UPB_DESCRIPTOR_TYPE_SINT32: + VARINT_CASE_EX(int32_t, upb_zzdecode_32, uint32_t); + case UPB_DESCRIPTOR_TYPE_SINT64: + VARINT_CASE_EX(int64_t, upb_zzdecode_64, uint64_t); + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + const upb_msglayout *subm; + upb_msg *submsg = upb_addmsg(frame, field, &subm); + CHK(submsg); + return upb_decode_msgfield(d, submsg, subm, len); + } + case UPB_DESCRIPTOR_TYPE_GROUP: + return upb_append_unknown(d, frame); + } +#undef VARINT_CASE + UPB_UNREACHABLE(); +} + +static bool upb_decode_delimitedfield(upb_decstate *d, upb_decframe *frame, + const upb_msglayout_field *field) { + int len; + + CHK(upb_decode_string(&d->ptr, d->limit, &len)); + + if (field->label == UPB_LABEL_REPEATED) { + return upb_decode_toarray(d, frame, field, len); + } else { + switch (field->descriptortype) { + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview str = upb_decode_strfield(d, len); + CHK(upb_decode_addval(frame, field, &str, sizeof(str))); + break; + } + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + const upb_msglayout *subm; + upb_msg *submsg = upb_getorcreatemsg(frame, field, &subm); + CHK(submsg); + CHK(upb_decode_msgfield(d, submsg, subm, len)); + break; + } + default: + /* TODO(haberman): should we accept the last element of a packed? */ + d->ptr += len; + return upb_append_unknown(d, frame); + } + upb_decode_setpresent(frame, field); + return true; + } +} + +static const upb_msglayout_field *upb_find_field(const upb_msglayout *l, + uint32_t field_number) { + /* Lots of optimization opportunities here. */ + int i; + for (i = 0; i < l->field_count; i++) { + if (l->fields[i].number == field_number) { + return &l->fields[i]; + } + } + + return NULL; /* Unknown field. */ +} + +static bool upb_decode_field(upb_decstate *d, upb_decframe *frame) { + uint32_t tag; + const upb_msglayout_field *field; + int field_number; + + d->field_start = d->ptr; + CHK(upb_decode_varint32(&d->ptr, d->limit, &tag)); + field_number = tag >> 3; + field = upb_find_field(frame->layout, field_number); + + if (field) { + switch (tag & 7) { + case UPB_WIRE_TYPE_VARINT: + return upb_decode_varintfield(d, frame, field); + case UPB_WIRE_TYPE_32BIT: + return upb_decode_32bitfield(d, frame, field); + case UPB_WIRE_TYPE_64BIT: + return upb_decode_64bitfield(d, frame, field); + case UPB_WIRE_TYPE_DELIMITED: + return upb_decode_delimitedfield(d, frame, field); + case UPB_WIRE_TYPE_START_GROUP: { + const upb_msglayout *layout; + upb_msg *group; + + if (field->label == UPB_LABEL_REPEATED) { + group = upb_addmsg(frame, field, &layout); + } else { + group = upb_getorcreatemsg(frame, field, &layout); + } + + return upb_decode_groupfield(d, group, layout, field_number); + } + case UPB_WIRE_TYPE_END_GROUP: + d->end_group = field_number; + return true; + default: + CHK(false); + } + } else { + CHK(field_number != 0); + CHK(upb_skip_unknownfielddata(d, tag, -1)); + CHK(upb_append_unknown(d, frame)); + return true; + } +} + +static bool upb_decode_message(upb_decstate *d, char *msg, const upb_msglayout *l) { + upb_decframe frame; + frame.msg = msg; + frame.layout = l; + frame.state = d; + + while (d->ptr < d->limit) { + CHK(upb_decode_field(d, &frame)); + } + + return true; +} + +bool upb_decode(const char *buf, size_t size, void *msg, const upb_msglayout *l, + upb_arena *arena) { + upb_decstate state; + state.ptr = buf; + state.limit = buf + size; + state.arena = arena; + state.depth = 64; + state.end_group = 0; + + CHK(upb_decode_message(&state, msg, l)); + return state.end_group == 0; +} + +#undef CHK diff --git a/upb/decode.h b/upb/decode.h new file mode 100644 index 00000000000..9de8638de5c --- /dev/null +++ b/upb/decode.h @@ -0,0 +1,21 @@ +/* +** upb_decode: parsing into a upb_msg using a upb_msglayout. +*/ + +#ifndef UPB_DECODE_H_ +#define UPB_DECODE_H_ + +#include "upb/msg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool upb_decode(const char *buf, size_t size, upb_msg *msg, + const upb_msglayout *l, upb_arena *arena); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_DECODE_H_ */ diff --git a/upb/def.c b/upb/def.c new file mode 100644 index 00000000000..98aead6b674 --- /dev/null +++ b/upb/def.c @@ -0,0 +1,1756 @@ + +#include "upb/def.h" + +#include +#include +#include +#include +#include "google/protobuf/descriptor.upb.h" + +#include "upb/port_def.inc" + +typedef struct { + size_t len; + char str[1]; /* Null-terminated string data follows. */ +} str_t; + +static str_t *newstr(upb_alloc *alloc, const char *data, size_t len) { + str_t *ret = upb_malloc(alloc, sizeof(*ret) + len); + if (!ret) return NULL; + ret->len = len; + memcpy(ret->str, data, len); + ret->str[len] = '\0'; + return ret; +} + +struct upb_fielddef { + const upb_filedef *file; + const upb_msgdef *msgdef; + const char *full_name; + union { + int64_t sint; + uint64_t uint; + double dbl; + float flt; + bool boolean; + str_t *str; + } defaultval; + const upb_oneofdef *oneof; + union { + const upb_msgdef *msgdef; + const upb_enumdef *enumdef; + const google_protobuf_FieldDescriptorProto *unresolved; + } sub; + uint32_t number_; + uint32_t index_; + uint32_t selector_base; /* Used to index into a upb::Handlers table. */ + bool is_extension_; + bool lazy_; + bool packed_; + upb_descriptortype_t type_; + upb_label_t label_; +}; + +struct upb_msgdef { + const upb_filedef *file; + const char *full_name; + uint32_t selector_count; + uint32_t submsg_field_count; + + /* Tables for looking up fields by number and name. */ + upb_inttable itof; + upb_strtable ntof; + + const upb_fielddef *fields; + const upb_oneofdef *oneofs; + int field_count; + int oneof_count; + + /* Is this a map-entry message? */ + bool map_entry; + upb_wellknowntype_t well_known_type; + + /* TODO(haberman): proper extension ranges (there can be multiple). */ +}; + +struct upb_enumdef { + const upb_filedef *file; + const char *full_name; + upb_strtable ntoi; + upb_inttable iton; + int32_t defaultval; +}; + +struct upb_oneofdef { + const upb_msgdef *parent; + const char *full_name; + uint32_t index; + upb_strtable ntof; + upb_inttable itof; +}; + +struct upb_filedef { + const char *name; + const char *package; + const char *phpprefix; + const char *phpnamespace; + upb_syntax_t syntax; + + const upb_filedef **deps; + const upb_msgdef *msgs; + const upb_enumdef *enums; + const upb_fielddef *exts; + + int dep_count; + int msg_count; + int enum_count; + int ext_count; +}; + +struct upb_symtab { + upb_arena *arena; + upb_strtable syms; /* full_name -> packed def ptr */ + upb_strtable files; /* file_name -> upb_filedef* */ +}; + +/* Inside a symtab we store tagged pointers to specific def types. */ +typedef enum { + UPB_DEFTYPE_MSG = 0, + UPB_DEFTYPE_ENUM = 1, + UPB_DEFTYPE_FIELD = 2, + UPB_DEFTYPE_ONEOF = 3 +} upb_deftype_t; + +static const void *unpack_def(upb_value v, upb_deftype_t type) { + uintptr_t num = (uintptr_t)upb_value_getconstptr(v); + return (num & 3) == type ? (const void*)(num & ~3) : NULL; +} + +static upb_value pack_def(const void *ptr, upb_deftype_t type) { + uintptr_t num = (uintptr_t)ptr | type; + return upb_value_constptr((const void*)num); +} + +/* isalpha() etc. from are locale-dependent, which we don't want. */ +static bool upb_isbetween(char c, char low, char high) { + return c >= low && c <= high; +} + +static bool upb_isletter(char c) { + return upb_isbetween(c, 'A', 'Z') || upb_isbetween(c, 'a', 'z') || c == '_'; +} + +static bool upb_isalphanum(char c) { + return upb_isletter(c) || upb_isbetween(c, '0', '9'); +} + +static bool upb_isident(upb_strview name, bool full, upb_status *s) { + const char *str = name.data; + size_t len = name.size; + bool start = true; + size_t i; + for (i = 0; i < len; i++) { + char c = str[i]; + if (c == '.') { + if (start || !full) { + upb_status_seterrf(s, "invalid name: unexpected '.' (%s)", str); + return false; + } + start = true; + } else if (start) { + if (!upb_isletter(c)) { + upb_status_seterrf( + s, "invalid name: path components must start with a letter (%s)", + str); + return false; + } + start = false; + } else { + if (!upb_isalphanum(c)) { + upb_status_seterrf(s, "invalid name: non-alphanumeric character (%s)", + str); + return false; + } + } + } + return !start; +} + +static const char *shortdefname(const char *fullname) { + const char *p; + + if (fullname == NULL) { + return NULL; + } else if ((p = strrchr(fullname, '.')) == NULL) { + /* No '.' in the name, return the full string. */ + return fullname; + } else { + /* Return one past the last '.'. */ + return p + 1; + } +} + +/* All submessage fields are lower than all other fields. + * Secondly, fields are increasing in order. */ +uint32_t field_rank(const upb_fielddef *f) { + uint32_t ret = upb_fielddef_number(f); + const uint32_t high_bit = 1 << 30; + UPB_ASSERT(ret < high_bit); + if (!upb_fielddef_issubmsg(f)) + ret |= high_bit; + return ret; +} + +int cmp_fields(const void *p1, const void *p2) { + const upb_fielddef *f1 = *(upb_fielddef*const*)p1; + const upb_fielddef *f2 = *(upb_fielddef*const*)p2; + return field_rank(f1) - field_rank(f2); +} + +/* A few implementation details of handlers. We put these here to avoid + * a def -> handlers dependency. */ + +#define UPB_STATIC_SELECTOR_COUNT 3 /* Warning: also in upb/handlers.h. */ + +static uint32_t upb_handlers_selectorbaseoffset(const upb_fielddef *f) { + return upb_fielddef_isseq(f) ? 2 : 0; +} + +static uint32_t upb_handlers_selectorcount(const upb_fielddef *f) { + uint32_t ret = 1; + if (upb_fielddef_isseq(f)) ret += 2; /* STARTSEQ/ENDSEQ */ + if (upb_fielddef_isstring(f)) ret += 2; /* [STRING]/STARTSTR/ENDSTR */ + if (upb_fielddef_issubmsg(f)) { + /* ENDSUBMSG (STARTSUBMSG is at table beginning) */ + ret += 0; + if (upb_fielddef_lazy(f)) { + /* STARTSTR/ENDSTR/STRING (for lazy) */ + ret += 3; + } + } + return ret; +} + +static bool assign_msg_indices(upb_msgdef *m, upb_status *s) { + /* Sort fields. upb internally relies on UPB_TYPE_MESSAGE fields having the + * lowest indexes, but we do not publicly guarantee this. */ + upb_msg_field_iter j; + upb_msg_oneof_iter k; + int i; + uint32_t selector; + int n = upb_msgdef_numfields(m); + upb_fielddef **fields; + + if (n == 0) { + m->selector_count = UPB_STATIC_SELECTOR_COUNT; + m->submsg_field_count = 0; + return true; + } + + fields = upb_gmalloc(n * sizeof(*fields)); + if (!fields) { + upb_status_setoom(s); + return false; + } + + m->submsg_field_count = 0; + for(i = 0, upb_msg_field_begin(&j, m); + !upb_msg_field_done(&j); + upb_msg_field_next(&j), i++) { + upb_fielddef *f = upb_msg_iter_field(&j); + UPB_ASSERT(f->msgdef == m); + if (upb_fielddef_issubmsg(f)) { + m->submsg_field_count++; + } + fields[i] = f; + } + + qsort(fields, n, sizeof(*fields), cmp_fields); + + selector = UPB_STATIC_SELECTOR_COUNT + m->submsg_field_count; + for (i = 0; i < n; i++) { + upb_fielddef *f = fields[i]; + f->index_ = i; + f->selector_base = selector + upb_handlers_selectorbaseoffset(f); + selector += upb_handlers_selectorcount(f); + } + m->selector_count = selector; + + for(upb_msg_oneof_begin(&k, m), i = 0; + !upb_msg_oneof_done(&k); + upb_msg_oneof_next(&k), i++) { + upb_oneofdef *o = (upb_oneofdef*)upb_msg_iter_oneof(&k); + o->index = i; + } + + upb_gfree(fields); + return true; +} + +static void assign_msg_wellknowntype(upb_msgdef *m) { + const char *name = upb_msgdef_fullname(m); + if (name == NULL) { + m->well_known_type = UPB_WELLKNOWN_UNSPECIFIED; + return; + } + if (!strcmp(name, "google.protobuf.Any")) { + m->well_known_type = UPB_WELLKNOWN_ANY; + } else if (!strcmp(name, "google.protobuf.FieldMask")) { + m->well_known_type = UPB_WELLKNOWN_FIELDMASK; + } else if (!strcmp(name, "google.protobuf.Duration")) { + m->well_known_type = UPB_WELLKNOWN_DURATION; + } else if (!strcmp(name, "google.protobuf.Timestamp")) { + m->well_known_type = UPB_WELLKNOWN_TIMESTAMP; + } else if (!strcmp(name, "google.protobuf.DoubleValue")) { + m->well_known_type = UPB_WELLKNOWN_DOUBLEVALUE; + } else if (!strcmp(name, "google.protobuf.FloatValue")) { + m->well_known_type = UPB_WELLKNOWN_FLOATVALUE; + } else if (!strcmp(name, "google.protobuf.Int64Value")) { + m->well_known_type = UPB_WELLKNOWN_INT64VALUE; + } else if (!strcmp(name, "google.protobuf.UInt64Value")) { + m->well_known_type = UPB_WELLKNOWN_UINT64VALUE; + } else if (!strcmp(name, "google.protobuf.Int32Value")) { + m->well_known_type = UPB_WELLKNOWN_INT32VALUE; + } else if (!strcmp(name, "google.protobuf.UInt32Value")) { + m->well_known_type = UPB_WELLKNOWN_UINT32VALUE; + } else if (!strcmp(name, "google.protobuf.BoolValue")) { + m->well_known_type = UPB_WELLKNOWN_BOOLVALUE; + } else if (!strcmp(name, "google.protobuf.StringValue")) { + m->well_known_type = UPB_WELLKNOWN_STRINGVALUE; + } else if (!strcmp(name, "google.protobuf.BytesValue")) { + m->well_known_type = UPB_WELLKNOWN_BYTESVALUE; + } else if (!strcmp(name, "google.protobuf.Value")) { + m->well_known_type = UPB_WELLKNOWN_VALUE; + } else if (!strcmp(name, "google.protobuf.ListValue")) { + m->well_known_type = UPB_WELLKNOWN_LISTVALUE; + } else if (!strcmp(name, "google.protobuf.Struct")) { + m->well_known_type = UPB_WELLKNOWN_STRUCT; + } else { + m->well_known_type = UPB_WELLKNOWN_UNSPECIFIED; + } +} + + +/* upb_enumdef ****************************************************************/ + +const char *upb_enumdef_fullname(const upb_enumdef *e) { + return e->full_name; +} + +const char *upb_enumdef_name(const upb_enumdef *e) { + return shortdefname(e->full_name); +} + +const upb_filedef *upb_enumdef_file(const upb_enumdef *e) { + return e->file; +} + +int32_t upb_enumdef_default(const upb_enumdef *e) { + UPB_ASSERT(upb_enumdef_iton(e, e->defaultval)); + return e->defaultval; +} + +int upb_enumdef_numvals(const upb_enumdef *e) { + return upb_strtable_count(&e->ntoi); +} + +void upb_enum_begin(upb_enum_iter *i, const upb_enumdef *e) { + /* We iterate over the ntoi table, to account for duplicate numbers. */ + upb_strtable_begin(i, &e->ntoi); +} + +void upb_enum_next(upb_enum_iter *iter) { upb_strtable_next(iter); } +bool upb_enum_done(upb_enum_iter *iter) { return upb_strtable_done(iter); } + +bool upb_enumdef_ntoi(const upb_enumdef *def, const char *name, + size_t len, int32_t *num) { + upb_value v; + if (!upb_strtable_lookup2(&def->ntoi, name, len, &v)) { + return false; + } + if (num) *num = upb_value_getint32(v); + return true; +} + +const char *upb_enumdef_iton(const upb_enumdef *def, int32_t num) { + upb_value v; + return upb_inttable_lookup32(&def->iton, num, &v) ? + upb_value_getcstr(v) : NULL; +} + +const char *upb_enum_iter_name(upb_enum_iter *iter) { + return upb_strtable_iter_key(iter); +} + +int32_t upb_enum_iter_number(upb_enum_iter *iter) { + return upb_value_getint32(upb_strtable_iter_value(iter)); +} + + +/* upb_fielddef ***************************************************************/ + +const char *upb_fielddef_fullname(const upb_fielddef *f) { + return f->full_name; +} + +upb_fieldtype_t upb_fielddef_type(const upb_fielddef *f) { + switch (f->type_) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + return UPB_TYPE_DOUBLE; + case UPB_DESCRIPTOR_TYPE_FLOAT: + return UPB_TYPE_FLOAT; + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_SINT64: + case UPB_DESCRIPTOR_TYPE_SFIXED64: + return UPB_TYPE_INT64; + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + case UPB_DESCRIPTOR_TYPE_SINT32: + return UPB_TYPE_INT32; + case UPB_DESCRIPTOR_TYPE_UINT64: + case UPB_DESCRIPTOR_TYPE_FIXED64: + return UPB_TYPE_UINT64; + case UPB_DESCRIPTOR_TYPE_UINT32: + case UPB_DESCRIPTOR_TYPE_FIXED32: + return UPB_TYPE_UINT32; + case UPB_DESCRIPTOR_TYPE_ENUM: + return UPB_TYPE_ENUM; + case UPB_DESCRIPTOR_TYPE_BOOL: + return UPB_TYPE_BOOL; + case UPB_DESCRIPTOR_TYPE_STRING: + return UPB_TYPE_STRING; + case UPB_DESCRIPTOR_TYPE_BYTES: + return UPB_TYPE_BYTES; + case UPB_DESCRIPTOR_TYPE_GROUP: + case UPB_DESCRIPTOR_TYPE_MESSAGE: + return UPB_TYPE_MESSAGE; + } + UPB_UNREACHABLE(); +} + +upb_descriptortype_t upb_fielddef_descriptortype(const upb_fielddef *f) { + return f->type_; +} + +uint32_t upb_fielddef_index(const upb_fielddef *f) { + return f->index_; +} + +upb_label_t upb_fielddef_label(const upb_fielddef *f) { + return f->label_; +} + +uint32_t upb_fielddef_number(const upb_fielddef *f) { + return f->number_; +} + +bool upb_fielddef_isextension(const upb_fielddef *f) { + return f->is_extension_; +} + +bool upb_fielddef_lazy(const upb_fielddef *f) { + return f->lazy_; +} + +bool upb_fielddef_packed(const upb_fielddef *f) { + return f->packed_; +} + +const char *upb_fielddef_name(const upb_fielddef *f) { + return shortdefname(f->full_name); +} + +uint32_t upb_fielddef_selectorbase(const upb_fielddef *f) { + return f->selector_base; +} + +size_t upb_fielddef_getjsonname(const upb_fielddef *f, char *buf, size_t len) { + const char *name = upb_fielddef_name(f); + size_t src, dst = 0; + bool ucase_next = false; + +#define WRITE(byte) \ + ++dst; \ + if (dst < len) buf[dst - 1] = byte; \ + else if (dst == len) buf[dst - 1] = '\0' + + if (!name) { + WRITE('\0'); + return 0; + } + + /* Implement the transformation as described in the spec: + * 1. upper case all letters after an underscore. + * 2. remove all underscores. + */ + for (src = 0; name[src]; src++) { + if (name[src] == '_') { + ucase_next = true; + continue; + } + + if (ucase_next) { + WRITE(toupper(name[src])); + ucase_next = false; + } else { + WRITE(name[src]); + } + } + + WRITE('\0'); + return dst; + +#undef WRITE +} + +const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f) { + return f->msgdef; +} + +const upb_oneofdef *upb_fielddef_containingoneof(const upb_fielddef *f) { + return f->oneof; +} + +static void chkdefaulttype(const upb_fielddef *f, int ctype) { + UPB_UNUSED(f); + UPB_UNUSED(ctype); +} + +int64_t upb_fielddef_defaultint64(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_INT64); + return f->defaultval.sint; +} + +int32_t upb_fielddef_defaultint32(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_INT32); + return f->defaultval.sint; +} + +uint64_t upb_fielddef_defaultuint64(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_UINT64); + return f->defaultval.uint; +} + +uint32_t upb_fielddef_defaultuint32(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_UINT32); + return f->defaultval.uint; +} + +bool upb_fielddef_defaultbool(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_BOOL); + return f->defaultval.boolean; +} + +float upb_fielddef_defaultfloat(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_FLOAT); + return f->defaultval.flt; +} + +double upb_fielddef_defaultdouble(const upb_fielddef *f) { + chkdefaulttype(f, UPB_TYPE_DOUBLE); + return f->defaultval.dbl; +} + +const char *upb_fielddef_defaultstr(const upb_fielddef *f, size_t *len) { + str_t *str = f->defaultval.str; + UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_STRING || + upb_fielddef_type(f) == UPB_TYPE_BYTES || + upb_fielddef_type(f) == UPB_TYPE_ENUM); + if (str) { + if (len) *len = str->len; + return str->str; + } else { + if (len) *len = 0; + return NULL; + } +} + +const upb_msgdef *upb_fielddef_msgsubdef(const upb_fielddef *f) { + UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_MESSAGE); + return f->sub.msgdef; +} + +const upb_enumdef *upb_fielddef_enumsubdef(const upb_fielddef *f) { + UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_ENUM); + return f->sub.enumdef; +} + +bool upb_fielddef_issubmsg(const upb_fielddef *f) { + return upb_fielddef_type(f) == UPB_TYPE_MESSAGE; +} + +bool upb_fielddef_isstring(const upb_fielddef *f) { + return upb_fielddef_type(f) == UPB_TYPE_STRING || + upb_fielddef_type(f) == UPB_TYPE_BYTES; +} + +bool upb_fielddef_isseq(const upb_fielddef *f) { + return upb_fielddef_label(f) == UPB_LABEL_REPEATED; +} + +bool upb_fielddef_isprimitive(const upb_fielddef *f) { + return !upb_fielddef_isstring(f) && !upb_fielddef_issubmsg(f); +} + +bool upb_fielddef_ismap(const upb_fielddef *f) { + return upb_fielddef_isseq(f) && upb_fielddef_issubmsg(f) && + upb_msgdef_mapentry(upb_fielddef_msgsubdef(f)); +} + +bool upb_fielddef_hassubdef(const upb_fielddef *f) { + return upb_fielddef_issubmsg(f) || upb_fielddef_type(f) == UPB_TYPE_ENUM; +} + +bool upb_fielddef_haspresence(const upb_fielddef *f) { + if (upb_fielddef_isseq(f)) return false; + if (upb_fielddef_issubmsg(f)) return true; + return f->file->syntax == UPB_SYNTAX_PROTO2; +} + +static bool between(int32_t x, int32_t low, int32_t high) { + return x >= low && x <= high; +} + +bool upb_fielddef_checklabel(int32_t label) { return between(label, 1, 3); } +bool upb_fielddef_checktype(int32_t type) { return between(type, 1, 11); } +bool upb_fielddef_checkintfmt(int32_t fmt) { return between(fmt, 1, 3); } + +bool upb_fielddef_checkdescriptortype(int32_t type) { + return between(type, 1, 18); +} + +/* upb_msgdef *****************************************************************/ + +const char *upb_msgdef_fullname(const upb_msgdef *m) { + return m->full_name; +} + +const upb_filedef *upb_msgdef_file(const upb_msgdef *m) { + return m->file; +} + +const char *upb_msgdef_name(const upb_msgdef *m) { + return shortdefname(m->full_name); +} + +upb_syntax_t upb_msgdef_syntax(const upb_msgdef *m) { + return m->file->syntax; +} + +size_t upb_msgdef_selectorcount(const upb_msgdef *m) { + return m->selector_count; +} + +uint32_t upb_msgdef_submsgfieldcount(const upb_msgdef *m) { + return m->submsg_field_count; +} + +const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i) { + upb_value val; + return upb_inttable_lookup32(&m->itof, i, &val) ? + upb_value_getconstptr(val) : NULL; +} + +const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name, + size_t len) { + upb_value val; + + if (!upb_strtable_lookup2(&m->ntof, name, len, &val)) { + return NULL; + } + + return unpack_def(val, UPB_DEFTYPE_FIELD); +} + +const upb_oneofdef *upb_msgdef_ntoo(const upb_msgdef *m, const char *name, + size_t len) { + upb_value val; + + if (!upb_strtable_lookup2(&m->ntof, name, len, &val)) { + return NULL; + } + + return unpack_def(val, UPB_DEFTYPE_ONEOF); +} + +bool upb_msgdef_lookupname(const upb_msgdef *m, const char *name, size_t len, + const upb_fielddef **f, const upb_oneofdef **o) { + upb_value val; + + if (!upb_strtable_lookup2(&m->ntof, name, len, &val)) { + return false; + } + + *o = unpack_def(val, UPB_DEFTYPE_ONEOF); + *f = unpack_def(val, UPB_DEFTYPE_FIELD); + UPB_ASSERT((*o != NULL) ^ (*f != NULL)); /* Exactly one of the two should be set. */ + return true; +} + +int upb_msgdef_numfields(const upb_msgdef *m) { + /* The number table contains only fields. */ + return upb_inttable_count(&m->itof); +} + +int upb_msgdef_numoneofs(const upb_msgdef *m) { + /* The name table includes oneofs, and the number table does not. */ + return upb_strtable_count(&m->ntof) - upb_inttable_count(&m->itof); +} + +bool upb_msgdef_mapentry(const upb_msgdef *m) { + return m->map_entry; +} + +upb_wellknowntype_t upb_msgdef_wellknowntype(const upb_msgdef *m) { + return m->well_known_type; +} + +bool upb_msgdef_isnumberwrapper(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type >= UPB_WELLKNOWN_DOUBLEVALUE && + type <= UPB_WELLKNOWN_UINT32VALUE; +} + +void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m) { + upb_inttable_begin(iter, &m->itof); +} + +void upb_msg_field_next(upb_msg_field_iter *iter) { upb_inttable_next(iter); } + +bool upb_msg_field_done(const upb_msg_field_iter *iter) { + return upb_inttable_done(iter); +} + +upb_fielddef *upb_msg_iter_field(const upb_msg_field_iter *iter) { + return (upb_fielddef *)upb_value_getconstptr(upb_inttable_iter_value(iter)); +} + +void upb_msg_field_iter_setdone(upb_msg_field_iter *iter) { + upb_inttable_iter_setdone(iter); +} + +bool upb_msg_field_iter_isequal(const upb_msg_field_iter * iter1, + const upb_msg_field_iter * iter2) { + return upb_inttable_iter_isequal(iter1, iter2); +} + +void upb_msg_oneof_begin(upb_msg_oneof_iter *iter, const upb_msgdef *m) { + upb_strtable_begin(iter, &m->ntof); + /* We need to skip past any initial fields. */ + while (!upb_strtable_done(iter) && + !unpack_def(upb_strtable_iter_value(iter), UPB_DEFTYPE_ONEOF)) { + upb_strtable_next(iter); + } +} + +void upb_msg_oneof_next(upb_msg_oneof_iter *iter) { + /* We need to skip past fields to return only oneofs. */ + do { + upb_strtable_next(iter); + } while (!upb_strtable_done(iter) && + !unpack_def(upb_strtable_iter_value(iter), UPB_DEFTYPE_ONEOF)); +} + +bool upb_msg_oneof_done(const upb_msg_oneof_iter *iter) { + return upb_strtable_done(iter); +} + +const upb_oneofdef *upb_msg_iter_oneof(const upb_msg_oneof_iter *iter) { + return unpack_def(upb_strtable_iter_value(iter), UPB_DEFTYPE_ONEOF); +} + +void upb_msg_oneof_iter_setdone(upb_msg_oneof_iter *iter) { + upb_strtable_iter_setdone(iter); +} + +bool upb_msg_oneof_iter_isequal(const upb_msg_oneof_iter *iter1, + const upb_msg_oneof_iter *iter2) { + return upb_strtable_iter_isequal(iter1, iter2); +} + +/* upb_oneofdef ***************************************************************/ + +const char *upb_oneofdef_name(const upb_oneofdef *o) { + return shortdefname(o->full_name); +} + +const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o) { + return o->parent; +} + +int upb_oneofdef_numfields(const upb_oneofdef *o) { + return upb_strtable_count(&o->ntof); +} + +uint32_t upb_oneofdef_index(const upb_oneofdef *o) { + return o->index; +} + +const upb_fielddef *upb_oneofdef_ntof(const upb_oneofdef *o, + const char *name, size_t length) { + upb_value val; + return upb_strtable_lookup2(&o->ntof, name, length, &val) ? + upb_value_getptr(val) : NULL; +} + +const upb_fielddef *upb_oneofdef_itof(const upb_oneofdef *o, uint32_t num) { + upb_value val; + return upb_inttable_lookup32(&o->itof, num, &val) ? + upb_value_getptr(val) : NULL; +} + +void upb_oneof_begin(upb_oneof_iter *iter, const upb_oneofdef *o) { + upb_inttable_begin(iter, &o->itof); +} + +void upb_oneof_next(upb_oneof_iter *iter) { + upb_inttable_next(iter); +} + +bool upb_oneof_done(upb_oneof_iter *iter) { + return upb_inttable_done(iter); +} + +upb_fielddef *upb_oneof_iter_field(const upb_oneof_iter *iter) { + return (upb_fielddef *)upb_value_getconstptr(upb_inttable_iter_value(iter)); +} + +void upb_oneof_iter_setdone(upb_oneof_iter *iter) { + upb_inttable_iter_setdone(iter); +} + +/* Code to build defs from descriptor protos. *********************************/ + +/* There is a question of how much validation to do here. It will be difficult + * to perfectly match the amount of validation performed by proto2. But since + * this code is used to directly build defs from Ruby (for example) we do need + * to validate important constraints like uniqueness of names and numbers. */ + +#define CHK(x) if (!(x)) { return false; } +#define CHK_OOM(x) if (!(x)) { upb_status_setoom(ctx->status); return false; } + +typedef struct { + const upb_symtab *symtab; + upb_filedef *file; /* File we are building. */ + upb_alloc *alloc; /* Allocate defs here. */ + upb_alloc *tmp; /* Alloc for addtab and any other tmp data. */ + upb_strtable *addtab; /* full_name -> packed def ptr for new defs. */ + upb_status *status; /* Record errors here. */ +} symtab_addctx; + +static char* strviewdup(const symtab_addctx *ctx, upb_strview view) { + return upb_strdup2(view.data, view.size, ctx->alloc); +} + +static bool streql2(const char *a, size_t n, const char *b) { + return n == strlen(b) && memcmp(a, b, n) == 0; +} + +static bool streql_view(upb_strview view, const char *b) { + return streql2(view.data, view.size, b); +} + +static const char *makefullname(const symtab_addctx *ctx, const char *prefix, + upb_strview name) { + if (prefix) { + /* ret = prefix + '.' + name; */ + size_t n = strlen(prefix); + char *ret = upb_malloc(ctx->alloc, n + name.size + 2); + CHK_OOM(ret); + strcpy(ret, prefix); + ret[n] = '.'; + memcpy(&ret[n + 1], name.data, name.size); + ret[n + 1 + name.size] = '\0'; + return ret; + } else { + return strviewdup(ctx, name); + } +} + +static bool symtab_add(const symtab_addctx *ctx, const char *name, + upb_value v) { + upb_value tmp; + if (upb_strtable_lookup(ctx->addtab, name, &tmp) || + upb_strtable_lookup(&ctx->symtab->syms, name, &tmp)) { + upb_status_seterrf(ctx->status, "duplicate symbol '%s'", name); + return false; + } + + CHK_OOM(upb_strtable_insert3(ctx->addtab, name, strlen(name), v, ctx->tmp)); + return true; +} + +/* Given a symbol and the base symbol inside which it is defined, find the + * symbol's definition in t. */ +static bool resolvename(const upb_strtable *t, const upb_fielddef *f, + const char *base, upb_strview sym, + upb_deftype_t type, upb_status *status, + const void **def) { + if(sym.size == 0) return NULL; + if(sym.data[0] == '.') { + /* Symbols starting with '.' are absolute, so we do a single lookup. + * Slice to omit the leading '.' */ + upb_value v; + if (!upb_strtable_lookup2(t, sym.data + 1, sym.size - 1, &v)) { + return false; + } + + *def = unpack_def(v, type); + + if (!*def) { + upb_status_seterrf(status, + "type mismatch when resolving field %s, name %s", + f->full_name, sym.data); + return false; + } + + return true; + } else { + /* Remove components from base until we find an entry or run out. + * TODO: This branch is totally broken, but currently not used. */ + (void)base; + UPB_ASSERT(false); + return false; + } +} + +const void *symtab_resolve(const symtab_addctx *ctx, const upb_fielddef *f, + const char *base, upb_strview sym, + upb_deftype_t type) { + const void *ret; + if (!resolvename(ctx->addtab, f, base, sym, type, ctx->status, &ret) && + !resolvename(&ctx->symtab->syms, f, base, sym, type, ctx->status, &ret)) { + if (upb_ok(ctx->status)) { + upb_status_seterrf(ctx->status, "couldn't resolve name '%s'", sym.data); + } + return false; + } + return ret; +} + +static bool create_oneofdef( + const symtab_addctx *ctx, upb_msgdef *m, + const google_protobuf_OneofDescriptorProto *oneof_proto) { + upb_oneofdef *o; + upb_strview name = google_protobuf_OneofDescriptorProto_name(oneof_proto); + upb_value v; + + o = (upb_oneofdef*)&m->oneofs[m->oneof_count++]; + o->parent = m; + o->full_name = makefullname(ctx, m->full_name, name); + + v = pack_def(o, UPB_DEFTYPE_ONEOF); + CHK_OOM(symtab_add(ctx, o->full_name, v)); + CHK_OOM(upb_strtable_insert3(&m->ntof, name.data, name.size, v, ctx->alloc)); + + CHK_OOM(upb_inttable_init2(&o->itof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + CHK_OOM(upb_strtable_init2(&o->ntof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + + return true; +} + +static bool parse_default(const symtab_addctx *ctx, const char *str, size_t len, + upb_fielddef *f) { + char *end; + char nullz[64]; + errno = 0; + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + /* Standard C number parsing functions expect null-terminated strings. */ + if (len >= sizeof(nullz) - 1) { + return false; + } + memcpy(nullz, str, len); + nullz[len] = '\0'; + str = nullz; + break; + default: + break; + } + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: { + long val = strtol(str, &end, 0); + CHK(val <= INT32_MAX && val >= INT32_MIN && errno != ERANGE && !*end); + f->defaultval.sint = val; + break; + } + case UPB_TYPE_ENUM: { + const upb_enumdef *e = f->sub.enumdef; + int32_t val; + CHK(upb_enumdef_ntoi(e, str, len, &val)); + f->defaultval.sint = val; + break; + } + case UPB_TYPE_INT64: { + /* XXX: Need to write our own strtoll, since it's not available in c89. */ + long long val = strtol(str, &end, 0); + CHK(val <= INT64_MAX && val >= INT64_MIN && errno != ERANGE && !*end); + f->defaultval.sint = val; + break; + } + case UPB_TYPE_UINT32: { + unsigned long val = strtoul(str, &end, 0); + CHK(val <= UINT32_MAX && errno != ERANGE && !*end); + f->defaultval.uint = val; + break; + } + case UPB_TYPE_UINT64: { + /* XXX: Need to write our own strtoull, since it's not available in c89. */ + unsigned long long val = strtoul(str, &end, 0); + CHK(val <= UINT64_MAX && errno != ERANGE && !*end); + f->defaultval.uint = val; + break; + } + case UPB_TYPE_DOUBLE: { + double val = strtod(str, &end); + CHK(errno != ERANGE && !*end); + f->defaultval.dbl = val; + break; + } + case UPB_TYPE_FLOAT: { + /* XXX: Need to write our own strtof, since it's not available in c89. */ + float val = strtod(str, &end); + CHK(errno != ERANGE && !*end); + f->defaultval.flt = val; + break; + } + case UPB_TYPE_BOOL: { + if (streql2(str, len, "false")) { + f->defaultval.boolean = false; + } else if (streql2(str, len, "true")) { + f->defaultval.boolean = true; + } else { + return false; + } + break; + } + case UPB_TYPE_STRING: + f->defaultval.str = newstr(ctx->alloc, str, len); + break; + case UPB_TYPE_BYTES: + /* XXX: need to interpret the C-escaped value. */ + f->defaultval.str = newstr(ctx->alloc, str, len); + break; + case UPB_TYPE_MESSAGE: + /* Should not have a default value. */ + return false; + } + return true; +} + +static void set_default_default(const symtab_addctx *ctx, upb_fielddef *f) { + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_ENUM: + f->defaultval.sint = 0; + break; + case UPB_TYPE_UINT64: + case UPB_TYPE_UINT32: + f->defaultval.uint = 0; + break; + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + f->defaultval.dbl = 0; + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + f->defaultval.str = newstr(ctx->alloc, NULL, 0); + break; + case UPB_TYPE_BOOL: + f->defaultval.boolean = false; + break; + case UPB_TYPE_MESSAGE: + break; + } +} + +static bool create_fielddef( + const symtab_addctx *ctx, const char *prefix, upb_msgdef *m, + const google_protobuf_FieldDescriptorProto *field_proto) { + upb_alloc *alloc = ctx->alloc; + upb_fielddef *f; + const google_protobuf_FieldOptions *options; + upb_strview name; + const char *full_name; + const char *shortname; + uint32_t field_number; + + if (!google_protobuf_FieldDescriptorProto_has_name(field_proto)) { + upb_status_seterrmsg(ctx->status, "field has no name"); + return false; + } + + name = google_protobuf_FieldDescriptorProto_name(field_proto); + CHK(upb_isident(name, false, ctx->status)); + full_name = makefullname(ctx, prefix, name); + shortname = shortdefname(full_name); + + field_number = google_protobuf_FieldDescriptorProto_number(field_proto); + + if (field_number == 0 || field_number > UPB_MAX_FIELDNUMBER) { + upb_status_seterrf(ctx->status, "invalid field number (%u)", field_number); + return false; + } + + if (m) { + /* direct message field. */ + upb_value v, packed_v; + + f = (upb_fielddef*)&m->fields[m->field_count++]; + f->msgdef = m; + f->is_extension_ = false; + + packed_v = pack_def(f, UPB_DEFTYPE_FIELD); + v = upb_value_constptr(f); + + if (!upb_strtable_insert3(&m->ntof, name.data, name.size, packed_v, alloc)) { + upb_status_seterrf(ctx->status, "duplicate field name (%s)", shortname); + return false; + } + + if (!upb_inttable_insert2(&m->itof, field_number, v, alloc)) { + upb_status_seterrf(ctx->status, "duplicate field number (%u)", + field_number); + return false; + } + } else { + /* extension field. */ + f = (upb_fielddef*)&ctx->file->exts[ctx->file->ext_count]; + f->is_extension_ = true; + CHK_OOM(symtab_add(ctx, full_name, pack_def(f, UPB_DEFTYPE_FIELD))); + } + + f->full_name = full_name; + f->file = ctx->file; + f->type_ = (int)google_protobuf_FieldDescriptorProto_type(field_proto); + f->label_ = (int)google_protobuf_FieldDescriptorProto_label(field_proto); + f->number_ = field_number; + f->oneof = NULL; + + /* We can't resolve the subdef or (in the case of extensions) the containing + * message yet, because it may not have been defined yet. We stash a pointer + * to the field_proto until later when we can properly resolve it. */ + f->sub.unresolved = field_proto; + + if (f->label_ == UPB_LABEL_REQUIRED && f->file->syntax == UPB_SYNTAX_PROTO3) { + upb_status_seterrf(ctx->status, "proto3 fields cannot be required (%s)", + f->full_name); + return false; + } + + if (google_protobuf_FieldDescriptorProto_has_oneof_index(field_proto)) { + int oneof_index = + google_protobuf_FieldDescriptorProto_oneof_index(field_proto); + upb_oneofdef *oneof; + upb_value v = upb_value_constptr(f); + + if (upb_fielddef_label(f) != UPB_LABEL_OPTIONAL) { + upb_status_seterrf(ctx->status, + "fields in oneof must have OPTIONAL label (%s)", + f->full_name); + return false; + } + + if (!m) { + upb_status_seterrf(ctx->status, + "oneof_index provided for extension field (%s)", + f->full_name); + return false; + } + + if (oneof_index >= m->oneof_count) { + upb_status_seterrf(ctx->status, "oneof_index out of range (%s)", + f->full_name); + return false; + } + + oneof = (upb_oneofdef*)&m->oneofs[oneof_index]; + f->oneof = oneof; + + CHK(upb_inttable_insert2(&oneof->itof, f->number_, v, alloc)); + CHK(upb_strtable_insert3(&oneof->ntof, name.data, name.size, v, alloc)); + } else { + f->oneof = NULL; + } + + if (google_protobuf_FieldDescriptorProto_has_options(field_proto)) { + options = google_protobuf_FieldDescriptorProto_options(field_proto); + f->lazy_ = google_protobuf_FieldOptions_lazy(options); + f->packed_ = google_protobuf_FieldOptions_packed(options); + } else { + f->lazy_ = false; + f->packed_ = false; + } + + return true; +} + +static bool create_enumdef( + const symtab_addctx *ctx, const char *prefix, + const google_protobuf_EnumDescriptorProto *enum_proto) { + upb_enumdef *e; + const google_protobuf_EnumValueDescriptorProto *const *values; + upb_strview name; + size_t i, n; + + name = google_protobuf_EnumDescriptorProto_name(enum_proto); + CHK(upb_isident(name, false, ctx->status)); + + e = (upb_enumdef*)&ctx->file->enums[ctx->file->enum_count++]; + e->full_name = makefullname(ctx, prefix, name); + CHK_OOM(symtab_add(ctx, e->full_name, pack_def(e, UPB_DEFTYPE_ENUM))); + + CHK_OOM(upb_strtable_init2(&e->ntoi, UPB_CTYPE_INT32, ctx->alloc)); + CHK_OOM(upb_inttable_init2(&e->iton, UPB_CTYPE_CSTR, ctx->alloc)); + + e->file = ctx->file; + e->defaultval = 0; + + values = google_protobuf_EnumDescriptorProto_value(enum_proto, &n); + + if (n == 0) { + upb_status_seterrf(ctx->status, + "enums must contain at least one value (%s)", + e->full_name); + return false; + } + + for (i = 0; i < n; i++) { + const google_protobuf_EnumValueDescriptorProto *value = values[i]; + upb_strview name = google_protobuf_EnumValueDescriptorProto_name(value); + char *name2 = strviewdup(ctx, name); + int32_t num = google_protobuf_EnumValueDescriptorProto_number(value); + upb_value v = upb_value_int32(num); + + if (i == 0 && e->file->syntax == UPB_SYNTAX_PROTO3 && num != 0) { + upb_status_seterrf(ctx->status, + "for proto3, the first enum value must be zero (%s)", + e->full_name); + return false; + } + + if (upb_strtable_lookup(&e->ntoi, name2, NULL)) { + upb_status_seterrf(ctx->status, "duplicate enum label '%s'", name2); + return false; + } + + CHK_OOM(name2) + CHK_OOM( + upb_strtable_insert3(&e->ntoi, name2, strlen(name2), v, ctx->alloc)); + + if (!upb_inttable_lookup(&e->iton, num, NULL)) { + upb_value v = upb_value_cstr(name2); + CHK_OOM(upb_inttable_insert2(&e->iton, num, v, ctx->alloc)); + } + } + + upb_inttable_compact2(&e->iton, ctx->alloc); + + return true; +} + +static bool create_msgdef(const symtab_addctx *ctx, const char *prefix, + const google_protobuf_DescriptorProto *msg_proto) { + upb_msgdef *m; + const google_protobuf_MessageOptions *options; + const google_protobuf_OneofDescriptorProto *const *oneofs; + const google_protobuf_FieldDescriptorProto *const *fields; + const google_protobuf_EnumDescriptorProto *const *enums; + const google_protobuf_DescriptorProto *const *msgs; + size_t i, n; + upb_strview name; + + name = google_protobuf_DescriptorProto_name(msg_proto); + CHK(upb_isident(name, false, ctx->status)); + + m = (upb_msgdef*)&ctx->file->msgs[ctx->file->msg_count++]; + m->full_name = makefullname(ctx, prefix, name); + CHK_OOM(symtab_add(ctx, m->full_name, pack_def(m, UPB_DEFTYPE_MSG))); + + CHK_OOM(upb_inttable_init2(&m->itof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + CHK_OOM(upb_strtable_init2(&m->ntof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + + m->file = ctx->file; + m->map_entry = false; + + options = google_protobuf_DescriptorProto_options(msg_proto); + + if (options) { + m->map_entry = google_protobuf_MessageOptions_map_entry(options); + } + + oneofs = google_protobuf_DescriptorProto_oneof_decl(msg_proto, &n); + m->oneof_count = 0; + m->oneofs = upb_malloc(ctx->alloc, sizeof(*m->oneofs) * n); + for (i = 0; i < n; i++) { + CHK(create_oneofdef(ctx, m, oneofs[i])); + } + + fields = google_protobuf_DescriptorProto_field(msg_proto, &n); + m->field_count = 0; + m->fields = upb_malloc(ctx->alloc, sizeof(*m->fields) * n); + for (i = 0; i < n; i++) { + CHK(create_fielddef(ctx, m->full_name, m, fields[i])); + } + + CHK(assign_msg_indices(m, ctx->status)); + assign_msg_wellknowntype(m); + upb_inttable_compact2(&m->itof, ctx->alloc); + + /* This message is built. Now build nested messages and enums. */ + + enums = google_protobuf_DescriptorProto_enum_type(msg_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_enumdef(ctx, m->full_name, enums[i])); + } + + msgs = google_protobuf_DescriptorProto_nested_type(msg_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_msgdef(ctx, m->full_name, msgs[i])); + } + + return true; +} + +typedef struct { + int msg_count; + int enum_count; + int ext_count; +} decl_counts; + +static void count_types_in_msg(const google_protobuf_DescriptorProto *msg_proto, + decl_counts *counts) { + const google_protobuf_DescriptorProto *const *msgs; + size_t i, n; + + counts->msg_count++; + + msgs = google_protobuf_DescriptorProto_nested_type(msg_proto, &n); + for (i = 0; i < n; i++) { + count_types_in_msg(msgs[i], counts); + } + + google_protobuf_DescriptorProto_enum_type(msg_proto, &n); + counts->enum_count += n; + + google_protobuf_DescriptorProto_extension(msg_proto, &n); + counts->ext_count += n; +} + +static void count_types_in_file( + const google_protobuf_FileDescriptorProto *file_proto, + decl_counts *counts) { + const google_protobuf_DescriptorProto *const *msgs; + size_t i, n; + + msgs = google_protobuf_FileDescriptorProto_message_type(file_proto, &n); + for (i = 0; i < n; i++) { + count_types_in_msg(msgs[i], counts); + } + + google_protobuf_FileDescriptorProto_enum_type(file_proto, &n); + counts->enum_count += n; + + google_protobuf_FileDescriptorProto_extension(file_proto, &n); + counts->ext_count += n; +} + +static bool resolve_fielddef(const symtab_addctx *ctx, const char *prefix, + upb_fielddef *f) { + upb_strview name; + const google_protobuf_FieldDescriptorProto *field_proto = f->sub.unresolved; + + if (f->is_extension_) { + if (!google_protobuf_FieldDescriptorProto_has_extendee(field_proto)) { + upb_status_seterrf(ctx->status, + "extension for field '%s' had no extendee", + f->full_name); + return false; + } + + name = google_protobuf_FieldDescriptorProto_extendee(field_proto); + f->msgdef = symtab_resolve(ctx, f, prefix, name, UPB_DEFTYPE_MSG); + CHK(f->msgdef); + } + + if ((upb_fielddef_issubmsg(f) || f->type_ == UPB_DESCRIPTOR_TYPE_ENUM) && + !google_protobuf_FieldDescriptorProto_has_type_name(field_proto)) { + upb_status_seterrf(ctx->status, "field '%s' is missing type name", + f->full_name); + return false; + } + + name = google_protobuf_FieldDescriptorProto_type_name(field_proto); + + if (upb_fielddef_issubmsg(f)) { + f->sub.msgdef = symtab_resolve(ctx, f, prefix, name, UPB_DEFTYPE_MSG); + CHK(f->sub.msgdef); + } else if (f->type_ == UPB_DESCRIPTOR_TYPE_ENUM) { + f->sub.enumdef = symtab_resolve(ctx, f, prefix, name, UPB_DEFTYPE_ENUM); + CHK(f->sub.enumdef); + } + + /* Have to delay resolving of the default value until now because of the enum + * case, since enum defaults are specified with a label. */ + if (google_protobuf_FieldDescriptorProto_has_default_value(field_proto)) { + upb_strview defaultval = + google_protobuf_FieldDescriptorProto_default_value(field_proto); + + if (f->file->syntax == UPB_SYNTAX_PROTO3) { + upb_status_seterrf(ctx->status, + "proto3 fields cannot have explicit defaults (%s)", + f->full_name); + return false; + } + + if (upb_fielddef_issubmsg(f)) { + upb_status_seterrf(ctx->status, + "message fields cannot have explicit defaults (%s)", + f->full_name); + return false; + } + + if (!parse_default(ctx, defaultval.data, defaultval.size, f)) { + upb_status_seterrf(ctx->status, + "couldn't parse default '" UPB_STRVIEW_FORMAT + "' for field (%s)", + UPB_STRVIEW_ARGS(defaultval), f->full_name); + return false; + } + } else { + set_default_default(ctx, f); + } + + return true; +} + +static bool build_filedef( + const symtab_addctx *ctx, upb_filedef *file, + const google_protobuf_FileDescriptorProto *file_proto) { + upb_alloc *alloc = ctx->alloc; + const google_protobuf_FileOptions *file_options_proto; + const google_protobuf_DescriptorProto *const *msgs; + const google_protobuf_EnumDescriptorProto *const *enums; + const google_protobuf_FieldDescriptorProto *const *exts; + const upb_strview* strs; + size_t i, n; + decl_counts counts = {0}; + + count_types_in_file(file_proto, &counts); + + file->msgs = upb_malloc(alloc, sizeof(*file->msgs) * counts.msg_count); + file->enums = upb_malloc(alloc, sizeof(*file->enums) * counts.enum_count); + file->exts = upb_malloc(alloc, sizeof(*file->exts) * counts.ext_count); + + CHK_OOM(counts.msg_count == 0 || file->msgs); + CHK_OOM(counts.enum_count == 0 || file->enums); + CHK_OOM(counts.ext_count == 0 || file->exts); + + /* We increment these as defs are added. */ + file->msg_count = 0; + file->enum_count = 0; + file->ext_count = 0; + + if (!google_protobuf_FileDescriptorProto_has_name(file_proto)) { + upb_status_seterrmsg(ctx->status, "File has no name"); + return false; + } + + file->name = + strviewdup(ctx, google_protobuf_FileDescriptorProto_name(file_proto)); + file->phpprefix = NULL; + file->phpnamespace = NULL; + + if (google_protobuf_FileDescriptorProto_has_package(file_proto)) { + upb_strview package = + google_protobuf_FileDescriptorProto_package(file_proto); + CHK(upb_isident(package, true, ctx->status)); + file->package = strviewdup(ctx, package); + } else { + file->package = NULL; + } + + if (google_protobuf_FileDescriptorProto_has_syntax(file_proto)) { + upb_strview syntax = + google_protobuf_FileDescriptorProto_syntax(file_proto); + + if (streql_view(syntax, "proto2")) { + file->syntax = UPB_SYNTAX_PROTO2; + } else if (streql_view(syntax, "proto3")) { + file->syntax = UPB_SYNTAX_PROTO3; + } else { + upb_status_seterrf(ctx->status, "Invalid syntax '%s'", syntax); + return false; + } + } else { + file->syntax = UPB_SYNTAX_PROTO2; + } + + /* Read options. */ + file_options_proto = google_protobuf_FileDescriptorProto_options(file_proto); + if (file_options_proto) { + if (google_protobuf_FileOptions_has_php_class_prefix(file_options_proto)) { + file->phpprefix = strviewdup( + ctx, + google_protobuf_FileOptions_php_class_prefix(file_options_proto)); + } + if (google_protobuf_FileOptions_has_php_namespace(file_options_proto)) { + file->phpnamespace = strviewdup( + ctx, google_protobuf_FileOptions_php_namespace(file_options_proto)); + } + } + + /* Verify dependencies. */ + strs = google_protobuf_FileDescriptorProto_dependency(file_proto, &n); + file->deps = upb_malloc(alloc, sizeof(*file->deps) * n) ; + CHK_OOM(n == 0 || file->deps); + + for (i = 0; i < n; i++) { + upb_strview dep_name = strs[i]; + upb_value v; + if (!upb_strtable_lookup2(&ctx->symtab->files, dep_name.data, + dep_name.size, &v)) { + upb_status_seterrf(ctx->status, + "Depends on file '" UPB_STRVIEW_FORMAT + "', but it has not been loaded", + UPB_STRVIEW_ARGS(dep_name)); + return false; + } + file->deps[i] = upb_value_getconstptr(v); + } + + /* Create messages. */ + msgs = google_protobuf_FileDescriptorProto_message_type(file_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_msgdef(ctx, file->package, msgs[i])); + } + + /* Create enums. */ + enums = google_protobuf_FileDescriptorProto_enum_type(file_proto, &n); + for (i = 0; i < n; i++) { + CHK(create_enumdef(ctx, file->package, enums[i])); + } + + /* Create extensions. */ + exts = google_protobuf_FileDescriptorProto_extension(file_proto, &n); + file->exts = upb_malloc(alloc, sizeof(*file->exts) * n); + CHK_OOM(n == 0 || file->exts); + for (i = 0; i < n; i++) { + CHK(create_fielddef(ctx, file->package, NULL, exts[i])); + } + + /* Now that all names are in the table, resolve references. */ + for (i = 0; i < file->ext_count; i++) { + CHK(resolve_fielddef(ctx, file->package, (upb_fielddef*)&file->exts[i])); + } + + for (i = 0; i < file->msg_count; i++) { + const upb_msgdef *m = &file->msgs[i]; + int j; + for (j = 0; j < m->field_count; j++) { + CHK(resolve_fielddef(ctx, m->full_name, (upb_fielddef*)&m->fields[j])); + } + } + + return true; + } + +static bool upb_symtab_addtotabs(upb_symtab *s, symtab_addctx *ctx, + upb_status *status) { + const upb_filedef *file = ctx->file; + upb_alloc *alloc = upb_arena_alloc(s->arena); + upb_strtable_iter iter; + + CHK_OOM(upb_strtable_insert3(&s->files, file->name, strlen(file->name), + upb_value_constptr(file), alloc)); + + upb_strtable_begin(&iter, ctx->addtab); + for (; !upb_strtable_done(&iter); upb_strtable_next(&iter)) { + const char *key = upb_strtable_iter_key(&iter); + size_t keylen = upb_strtable_iter_keylength(&iter); + upb_value value = upb_strtable_iter_value(&iter); + CHK_OOM(upb_strtable_insert3(&s->syms, key, keylen, value, alloc)); + } + + return true; +} + +/* upb_filedef ****************************************************************/ + +const char *upb_filedef_name(const upb_filedef *f) { + return f->name; +} + +const char *upb_filedef_package(const upb_filedef *f) { + return f->package; +} + +const char *upb_filedef_phpprefix(const upb_filedef *f) { + return f->phpprefix; +} + +const char *upb_filedef_phpnamespace(const upb_filedef *f) { + return f->phpnamespace; +} + +upb_syntax_t upb_filedef_syntax(const upb_filedef *f) { + return f->syntax; +} + +int upb_filedef_msgcount(const upb_filedef *f) { + return f->msg_count; +} + +int upb_filedef_depcount(const upb_filedef *f) { + return f->dep_count; +} + +int upb_filedef_enumcount(const upb_filedef *f) { + return f->enum_count; +} + +const upb_filedef *upb_filedef_dep(const upb_filedef *f, int i) { + return i < 0 || i >= f->dep_count ? NULL : f->deps[i]; +} + +const upb_msgdef *upb_filedef_msg(const upb_filedef *f, int i) { + return i < 0 || i >= f->msg_count ? NULL : &f->msgs[i]; +} + +const upb_enumdef *upb_filedef_enum(const upb_filedef *f, int i) { + return i < 0 || i >= f->enum_count ? NULL : &f->enums[i]; +} + +void upb_symtab_free(upb_symtab *s) { + upb_arena_free(s->arena); + upb_gfree(s); +} + +upb_symtab *upb_symtab_new(void) { + upb_symtab *s = upb_gmalloc(sizeof(*s)); + upb_alloc *alloc; + + if (!s) { + return NULL; + } + + s->arena = upb_arena_new(); + alloc = upb_arena_alloc(s->arena); + + if (!upb_strtable_init2(&s->syms, UPB_CTYPE_CONSTPTR, alloc) || + !upb_strtable_init2(&s->files, UPB_CTYPE_CONSTPTR, alloc)) { + upb_arena_free(s->arena); + upb_gfree(s); + s = NULL; + } + return s; +} + +const upb_msgdef *upb_symtab_lookupmsg(const upb_symtab *s, const char *sym) { + upb_value v; + return upb_strtable_lookup(&s->syms, sym, &v) ? + unpack_def(v, UPB_DEFTYPE_MSG) : NULL; +} + +const upb_msgdef *upb_symtab_lookupmsg2(const upb_symtab *s, const char *sym, + size_t len) { + upb_value v; + return upb_strtable_lookup2(&s->syms, sym, len, &v) ? + unpack_def(v, UPB_DEFTYPE_MSG) : NULL; +} + +const upb_enumdef *upb_symtab_lookupenum(const upb_symtab *s, const char *sym) { + upb_value v; + return upb_strtable_lookup(&s->syms, sym, &v) ? + unpack_def(v, UPB_DEFTYPE_ENUM) : NULL; +} + +const upb_filedef *upb_symtab_lookupfile(const upb_symtab *s, const char *name) { + upb_value v; + return upb_strtable_lookup(&s->files, name, &v) ? upb_value_getconstptr(v) + : NULL; +} + +const upb_filedef *upb_symtab_addfile( + upb_symtab *s, const google_protobuf_FileDescriptorProto *file_proto, + upb_status *status) { + upb_arena *tmparena = upb_arena_new(); + upb_strtable addtab; + upb_alloc *alloc = upb_arena_alloc(s->arena); + upb_filedef *file = upb_malloc(alloc, sizeof(*file)); + bool ok; + symtab_addctx ctx; + + ctx.file = file; + ctx.symtab = s; + ctx.alloc = alloc; + ctx.tmp = upb_arena_alloc(tmparena); + ctx.addtab = &addtab; + ctx.status = status; + + ok = file && + upb_strtable_init2(&addtab, UPB_CTYPE_CONSTPTR, ctx.tmp) && + build_filedef(&ctx, file, file_proto) && + upb_symtab_addtotabs(s, &ctx, status); + + upb_arena_free(tmparena); + return ok ? file : NULL; +} + +/* Include here since we want most of this file to be stdio-free. */ +#include + +bool _upb_symtab_loaddefinit(upb_symtab *s, const upb_def_init *init) { + /* Since this function should never fail (it would indicate a bug in upb) we + * print errors to stderr instead of returning error status to the user. */ + upb_def_init **deps = init->deps; + google_protobuf_FileDescriptorProto *file; + upb_arena *arena; + upb_status status; + + upb_status_clear(&status); + + if (upb_strtable_lookup(&s->files, init->filename, NULL)) { + return true; + } + + arena = upb_arena_new(); + + for (; *deps; deps++) { + if (!_upb_symtab_loaddefinit(s, *deps)) goto err; + } + + file = google_protobuf_FileDescriptorProto_parse( + init->descriptor.data, init->descriptor.size, arena); + + if (!file) { + upb_status_seterrf( + &status, + "Failed to parse compiled-in descriptor for file '%s'. This should " + "never happen.", + init->filename); + goto err; + } + + if (!upb_symtab_addfile(s, file, &status)) goto err; + + upb_arena_free(arena); + return true; + +err: + fprintf(stderr, "Error loading compiled-in descriptor: %s\n", + upb_status_errmsg(&status)); + upb_arena_free(arena); + return false; +} + +#undef CHK +#undef CHK_OOM diff --git a/upb/def.h b/upb/def.h new file mode 100644 index 00000000000..9be285794ee --- /dev/null +++ b/upb/def.h @@ -0,0 +1,909 @@ +/* +** Defs are upb's internal representation of the constructs that can appear +** in a .proto file: +** +** - upb::MessageDefPtr (upb_msgdef): describes a "message" construct. +** - upb::FieldDefPtr (upb_fielddef): describes a message field. +** - upb::FileDefPtr (upb_filedef): describes a .proto file and its defs. +** - upb::EnumDefPtr (upb_enumdef): describes an enum. +** - upb::OneofDefPtr (upb_oneofdef): describes a oneof. +** +** TODO: definitions of services. +** +** This is a mixed C/C++ interface that offers a full API to both languages. +** See the top-level README for more information. +*/ + +#ifndef UPB_DEF_H_ +#define UPB_DEF_H_ + +#include "upb/upb.h" +#include "upb/table.int.h" +#include "google/protobuf/descriptor.upb.h" + +#ifdef __cplusplus +#include +#include +#include +#include + +namespace upb { +class EnumDefPtr; +class FieldDefPtr; +class FileDefPtr; +class MessageDefPtr; +class OneofDefPtr; +class SymbolTable; +} +#endif + +#include "upb/port_def.inc" + +struct upb_enumdef; +typedef struct upb_enumdef upb_enumdef; +struct upb_fielddef; +typedef struct upb_fielddef upb_fielddef; +struct upb_filedef; +typedef struct upb_filedef upb_filedef; +struct upb_msgdef; +typedef struct upb_msgdef upb_msgdef; +struct upb_oneofdef; +typedef struct upb_oneofdef upb_oneofdef; +struct upb_symtab; +typedef struct upb_symtab upb_symtab; + +typedef enum { + UPB_SYNTAX_PROTO2 = 2, + UPB_SYNTAX_PROTO3 = 3 +} upb_syntax_t; + +/* All the different kind of well known type messages. For simplicity of check, + * number wrappers and string wrappers are grouped together. Make sure the + * order and merber of these groups are not changed. + */ +typedef enum { + UPB_WELLKNOWN_UNSPECIFIED, + UPB_WELLKNOWN_ANY, + UPB_WELLKNOWN_FIELDMASK, + UPB_WELLKNOWN_DURATION, + UPB_WELLKNOWN_TIMESTAMP, + /* number wrappers */ + UPB_WELLKNOWN_DOUBLEVALUE, + UPB_WELLKNOWN_FLOATVALUE, + UPB_WELLKNOWN_INT64VALUE, + UPB_WELLKNOWN_UINT64VALUE, + UPB_WELLKNOWN_INT32VALUE, + UPB_WELLKNOWN_UINT32VALUE, + /* string wrappers */ + UPB_WELLKNOWN_STRINGVALUE, + UPB_WELLKNOWN_BYTESVALUE, + UPB_WELLKNOWN_BOOLVALUE, + UPB_WELLKNOWN_VALUE, + UPB_WELLKNOWN_LISTVALUE, + UPB_WELLKNOWN_STRUCT +} upb_wellknowntype_t; + +/* upb_fielddef ***************************************************************/ + +/* Maximum field number allowed for FieldDefs. This is an inherent limit of the + * protobuf wire format. */ +#define UPB_MAX_FIELDNUMBER ((1 << 29) - 1) + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_fielddef_fullname(const upb_fielddef *f); +upb_fieldtype_t upb_fielddef_type(const upb_fielddef *f); +upb_descriptortype_t upb_fielddef_descriptortype(const upb_fielddef *f); +upb_label_t upb_fielddef_label(const upb_fielddef *f); +uint32_t upb_fielddef_number(const upb_fielddef *f); +const char *upb_fielddef_name(const upb_fielddef *f); +bool upb_fielddef_isextension(const upb_fielddef *f); +bool upb_fielddef_lazy(const upb_fielddef *f); +bool upb_fielddef_packed(const upb_fielddef *f); +size_t upb_fielddef_getjsonname(const upb_fielddef *f, char *buf, size_t len); +const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f); +const upb_oneofdef *upb_fielddef_containingoneof(const upb_fielddef *f); +uint32_t upb_fielddef_index(const upb_fielddef *f); +bool upb_fielddef_issubmsg(const upb_fielddef *f); +bool upb_fielddef_isstring(const upb_fielddef *f); +bool upb_fielddef_isseq(const upb_fielddef *f); +bool upb_fielddef_isprimitive(const upb_fielddef *f); +bool upb_fielddef_ismap(const upb_fielddef *f); +int64_t upb_fielddef_defaultint64(const upb_fielddef *f); +int32_t upb_fielddef_defaultint32(const upb_fielddef *f); +uint64_t upb_fielddef_defaultuint64(const upb_fielddef *f); +uint32_t upb_fielddef_defaultuint32(const upb_fielddef *f); +bool upb_fielddef_defaultbool(const upb_fielddef *f); +float upb_fielddef_defaultfloat(const upb_fielddef *f); +double upb_fielddef_defaultdouble(const upb_fielddef *f); +const char *upb_fielddef_defaultstr(const upb_fielddef *f, size_t *len); +bool upb_fielddef_hassubdef(const upb_fielddef *f); +bool upb_fielddef_haspresence(const upb_fielddef *f); +const upb_msgdef *upb_fielddef_msgsubdef(const upb_fielddef *f); +const upb_enumdef *upb_fielddef_enumsubdef(const upb_fielddef *f); + +/* Internal only. */ +uint32_t upb_fielddef_selectorbase(const upb_fielddef *f); + +#ifdef __cplusplus +} /* extern "C" */ + +/* A upb_fielddef describes a single field in a message. It is most often + * found as a part of a upb_msgdef, but can also stand alone to represent + * an extension. */ +class upb::FieldDefPtr { + public: + FieldDefPtr() : ptr_(nullptr) {} + explicit FieldDefPtr(const upb_fielddef *ptr) : ptr_(ptr) {} + + const upb_fielddef* ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + typedef upb_fieldtype_t Type; + typedef upb_label_t Label; + typedef upb_descriptortype_t DescriptorType; + + const char* full_name() const { return upb_fielddef_fullname(ptr_); } + + Type type() const { return upb_fielddef_type(ptr_); } + Label label() const { return upb_fielddef_label(ptr_); } + const char* name() const { return upb_fielddef_name(ptr_); } + uint32_t number() const { return upb_fielddef_number(ptr_); } + bool is_extension() const { return upb_fielddef_isextension(ptr_); } + + /* Copies the JSON name for this field into the given buffer. Returns the + * actual size of the JSON name, including the NULL terminator. If the + * return value is 0, the JSON name is unset. If the return value is + * greater than len, the JSON name was truncated. The buffer is always + * NULL-terminated if len > 0. + * + * The JSON name always defaults to a camelCased version of the regular + * name. However if the regular name is unset, the JSON name will be unset + * also. + */ + size_t GetJsonName(char *buf, size_t len) const { + return upb_fielddef_getjsonname(ptr_, buf, len); + } + + /* Convenience version of the above function which copies the JSON name + * into the given string, returning false if the name is not set. */ + template + bool GetJsonName(T* str) { + str->resize(GetJsonName(NULL, 0)); + GetJsonName(&(*str)[0], str->size()); + return str->size() > 0; + } + + /* For UPB_TYPE_MESSAGE fields only where is_tag_delimited() == false, + * indicates whether this field should have lazy parsing handlers that yield + * the unparsed string for the submessage. + * + * TODO(haberman): I think we want to move this into a FieldOptions container + * when we add support for custom options (the FieldOptions struct will + * contain both regular FieldOptions like "lazy" *and* custom options). */ + bool lazy() const { return upb_fielddef_lazy(ptr_); } + + /* For non-string, non-submessage fields, this indicates whether binary + * protobufs are encoded in packed or non-packed format. + * + * TODO(haberman): see note above about putting options like this into a + * FieldOptions container. */ + bool packed() const { return upb_fielddef_packed(ptr_); } + + /* An integer that can be used as an index into an array of fields for + * whatever message this field belongs to. Guaranteed to be less than + * f->containing_type()->field_count(). May only be accessed once the def has + * been finalized. */ + uint32_t index() const { return upb_fielddef_index(ptr_); } + + /* The MessageDef to which this field belongs. + * + * If this field has been added to a MessageDef, that message can be retrieved + * directly (this is always the case for frozen FieldDefs). + * + * If the field has not yet been added to a MessageDef, you can set the name + * of the containing type symbolically instead. This is mostly useful for + * extensions, where the extension is declared separately from the message. */ + MessageDefPtr containing_type() const; + + /* The OneofDef to which this field belongs, or NULL if this field is not part + * of a oneof. */ + OneofDefPtr containing_oneof() const; + + /* The field's type according to the enum in descriptor.proto. This is not + * the same as UPB_TYPE_*, because it distinguishes between (for example) + * INT32 and SINT32, whereas our "type" enum does not. This return of + * descriptor_type() is a function of type(), integer_format(), and + * is_tag_delimited(). */ + DescriptorType descriptor_type() const { + return upb_fielddef_descriptortype(ptr_); + } + + /* Convenient field type tests. */ + bool IsSubMessage() const { return upb_fielddef_issubmsg(ptr_); } + bool IsString() const { return upb_fielddef_isstring(ptr_); } + bool IsSequence() const { return upb_fielddef_isseq(ptr_); } + bool IsPrimitive() const { return upb_fielddef_isprimitive(ptr_); } + bool IsMap() const { return upb_fielddef_ismap(ptr_); } + + /* Returns the non-string default value for this fielddef, which may either + * be something the client set explicitly or the "default default" (0 for + * numbers, empty for strings). The field's type indicates the type of the + * returned value, except for enum fields that are still mutable. + * + * Requires that the given function matches the field's current type. */ + int64_t default_int64() const { return upb_fielddef_defaultint64(ptr_); } + int32_t default_int32() const { return upb_fielddef_defaultint32(ptr_); } + uint64_t default_uint64() const { return upb_fielddef_defaultuint64(ptr_); } + uint32_t default_uint32() const { return upb_fielddef_defaultuint32(ptr_); } + bool default_bool() const { return upb_fielddef_defaultbool(ptr_); } + float default_float() const { return upb_fielddef_defaultfloat(ptr_); } + double default_double() const { return upb_fielddef_defaultdouble(ptr_); } + + /* The resulting string is always NULL-terminated. If non-NULL, the length + * will be stored in *len. */ + const char *default_string(size_t * len) const { + return upb_fielddef_defaultstr(ptr_, len); + } + + /* Returns the enum or submessage def for this field, if any. The field's + * type must match (ie. you may only call enum_subdef() for fields where + * type() == UPB_TYPE_ENUM). */ + EnumDefPtr enum_subdef() const; + MessageDefPtr message_subdef() const; + + private: + const upb_fielddef *ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_oneofdef ***************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef upb_inttable_iter upb_oneof_iter; + +const char *upb_oneofdef_name(const upb_oneofdef *o); +const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o); +int upb_oneofdef_numfields(const upb_oneofdef *o); +uint32_t upb_oneofdef_index(const upb_oneofdef *o); + +/* Oneof lookups: + * - ntof: look up a field by name. + * - ntofz: look up a field by name (as a null-terminated string). + * - itof: look up a field by number. */ +const upb_fielddef *upb_oneofdef_ntof(const upb_oneofdef *o, + const char *name, size_t length); +UPB_INLINE const upb_fielddef *upb_oneofdef_ntofz(const upb_oneofdef *o, + const char *name) { + return upb_oneofdef_ntof(o, name, strlen(name)); +} +const upb_fielddef *upb_oneofdef_itof(const upb_oneofdef *o, uint32_t num); + +/* upb_oneof_iter i; + * for(upb_oneof_begin(&i, e); !upb_oneof_done(&i); upb_oneof_next(&i)) { + * // ... + * } + */ +void upb_oneof_begin(upb_oneof_iter *iter, const upb_oneofdef *o); +void upb_oneof_next(upb_oneof_iter *iter); +bool upb_oneof_done(upb_oneof_iter *iter); +upb_fielddef *upb_oneof_iter_field(const upb_oneof_iter *iter); +void upb_oneof_iter_setdone(upb_oneof_iter *iter); +bool upb_oneof_iter_isequal(const upb_oneof_iter *iter1, + const upb_oneof_iter *iter2); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Class that represents a oneof. */ +class upb::OneofDefPtr { + public: + OneofDefPtr() : ptr_(nullptr) {} + explicit OneofDefPtr(const upb_oneofdef *ptr) : ptr_(ptr) {} + + const upb_oneofdef* ptr() const { return ptr_; } + explicit operator bool() { return ptr_ != nullptr; } + + /* Returns the MessageDef that owns this OneofDef. */ + MessageDefPtr containing_type() const; + + /* Returns the name of this oneof. This is the name used to look up the oneof + * by name once added to a message def. */ + const char* name() const { return upb_oneofdef_name(ptr_); } + + /* Returns the number of fields currently defined in the oneof. */ + int field_count() const { return upb_oneofdef_numfields(ptr_); } + + /* Looks up by name. */ + FieldDefPtr FindFieldByName(const char *name, size_t len) const { + return FieldDefPtr(upb_oneofdef_ntof(ptr_, name, len)); + } + FieldDefPtr FindFieldByName(const char* name) const { + return FieldDefPtr(upb_oneofdef_ntofz(ptr_, name)); + } + + template + FieldDefPtr FindFieldByName(const T& str) const { + return FindFieldByName(str.c_str(), str.size()); + } + + /* Looks up by tag number. */ + FieldDefPtr FindFieldByNumber(uint32_t num) const { + return FieldDefPtr(upb_oneofdef_itof(ptr_, num)); + } + + class const_iterator + : public std::iterator { + public: + void operator++() { upb_oneof_next(&iter_); } + + FieldDefPtr operator*() const { + return FieldDefPtr(upb_oneof_iter_field(&iter_)); + } + + bool operator!=(const const_iterator& other) const { + return !upb_oneof_iter_isequal(&iter_, &other.iter_); + } + + bool operator==(const const_iterator& other) const { + return upb_oneof_iter_isequal(&iter_, &other.iter_); + } + + private: + friend class OneofDefPtr; + + const_iterator() {} + explicit const_iterator(OneofDefPtr o) { + upb_oneof_begin(&iter_, o.ptr()); + } + static const_iterator end() { + const_iterator iter; + upb_oneof_iter_setdone(&iter.iter_); + return iter; + } + + upb_oneof_iter iter_; + }; + + const_iterator begin() const { return const_iterator(*this); } + const_iterator end() const { return const_iterator::end(); } + + private: + const upb_oneofdef *ptr_; +}; + +inline upb::OneofDefPtr upb::FieldDefPtr::containing_oneof() const { + return OneofDefPtr(upb_fielddef_containingoneof(ptr_)); +} + +#endif /* __cplusplus */ + +/* upb_msgdef *****************************************************************/ + +typedef upb_inttable_iter upb_msg_field_iter; +typedef upb_strtable_iter upb_msg_oneof_iter; + +/* Well-known field tag numbers for map-entry messages. */ +#define UPB_MAPENTRY_KEY 1 +#define UPB_MAPENTRY_VALUE 2 + +/* Well-known field tag numbers for Any messages. */ +#define UPB_ANY_TYPE 1 +#define UPB_ANY_VALUE 2 + +/* Well-known field tag numbers for timestamp messages. */ +#define UPB_DURATION_SECONDS 1 +#define UPB_DURATION_NANOS 2 + +/* Well-known field tag numbers for duration messages. */ +#define UPB_TIMESTAMP_SECONDS 1 +#define UPB_TIMESTAMP_NANOS 2 + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_msgdef_fullname(const upb_msgdef *m); +const upb_filedef *upb_msgdef_file(const upb_msgdef *m); +const char *upb_msgdef_name(const upb_msgdef *m); +int upb_msgdef_numoneofs(const upb_msgdef *m); +upb_syntax_t upb_msgdef_syntax(const upb_msgdef *m); +bool upb_msgdef_mapentry(const upb_msgdef *m); +upb_wellknowntype_t upb_msgdef_wellknowntype(const upb_msgdef *m); +bool upb_msgdef_isnumberwrapper(const upb_msgdef *m); +bool upb_msgdef_setsyntax(upb_msgdef *m, upb_syntax_t syntax); +const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i); +const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name, + size_t len); +const upb_oneofdef *upb_msgdef_ntoo(const upb_msgdef *m, const char *name, + size_t len); +int upb_msgdef_numfields(const upb_msgdef *m); +int upb_msgdef_numoneofs(const upb_msgdef *m); + +UPB_INLINE const upb_oneofdef *upb_msgdef_ntooz(const upb_msgdef *m, + const char *name) { + return upb_msgdef_ntoo(m, name, strlen(name)); +} + +UPB_INLINE const upb_fielddef *upb_msgdef_ntofz(const upb_msgdef *m, + const char *name) { + return upb_msgdef_ntof(m, name, strlen(name)); +} + +/* Internal-only. */ +size_t upb_msgdef_selectorcount(const upb_msgdef *m); +uint32_t upb_msgdef_submsgfieldcount(const upb_msgdef *m); + +/* Lookup of either field or oneof by name. Returns whether either was found. + * If the return is true, then the found def will be set, and the non-found + * one set to NULL. */ +bool upb_msgdef_lookupname(const upb_msgdef *m, const char *name, size_t len, + const upb_fielddef **f, const upb_oneofdef **o); + +UPB_INLINE bool upb_msgdef_lookupnamez(const upb_msgdef *m, const char *name, + const upb_fielddef **f, + const upb_oneofdef **o) { + return upb_msgdef_lookupname(m, name, strlen(name), f, o); +} + +/* Iteration over fields and oneofs. For example: + * + * upb_msg_field_iter i; + * for(upb_msg_field_begin(&i, m); + * !upb_msg_field_done(&i); + * upb_msg_field_next(&i)) { + * upb_fielddef *f = upb_msg_iter_field(&i); + * // ... + * } + * + * For C we don't have separate iterators for const and non-const. + * It is the caller's responsibility to cast the upb_fielddef* to + * const if the upb_msgdef* is const. */ +void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m); +void upb_msg_field_next(upb_msg_field_iter *iter); +bool upb_msg_field_done(const upb_msg_field_iter *iter); +upb_fielddef *upb_msg_iter_field(const upb_msg_field_iter *iter); +void upb_msg_field_iter_setdone(upb_msg_field_iter *iter); +bool upb_msg_field_iter_isequal(const upb_msg_field_iter * iter1, + const upb_msg_field_iter * iter2); + +/* Similar to above, we also support iterating through the oneofs in a + * msgdef. */ +void upb_msg_oneof_begin(upb_msg_oneof_iter * iter, const upb_msgdef *m); +void upb_msg_oneof_next(upb_msg_oneof_iter * iter); +bool upb_msg_oneof_done(const upb_msg_oneof_iter *iter); +const upb_oneofdef *upb_msg_iter_oneof(const upb_msg_oneof_iter *iter); +void upb_msg_oneof_iter_setdone(upb_msg_oneof_iter * iter); +bool upb_msg_oneof_iter_isequal(const upb_msg_oneof_iter *iter1, + const upb_msg_oneof_iter *iter2); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Structure that describes a single .proto message type. */ +class upb::MessageDefPtr { + public: + MessageDefPtr() : ptr_(nullptr) {} + explicit MessageDefPtr(const upb_msgdef *ptr) : ptr_(ptr) {} + + const upb_msgdef *ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + const char* full_name() const { return upb_msgdef_fullname(ptr_); } + const char* name() const { return upb_msgdef_name(ptr_); } + + /* The number of fields that belong to the MessageDef. */ + int field_count() const { return upb_msgdef_numfields(ptr_); } + + /* The number of oneofs that belong to the MessageDef. */ + int oneof_count() const { return upb_msgdef_numoneofs(ptr_); } + + upb_syntax_t syntax() const { return upb_msgdef_syntax(ptr_); } + + /* These return null pointers if the field is not found. */ + FieldDefPtr FindFieldByNumber(uint32_t number) const { + return FieldDefPtr(upb_msgdef_itof(ptr_, number)); + } + FieldDefPtr FindFieldByName(const char* name, size_t len) const { + return FieldDefPtr(upb_msgdef_ntof(ptr_, name, len)); + } + FieldDefPtr FindFieldByName(const char *name) const { + return FieldDefPtr(upb_msgdef_ntofz(ptr_, name)); + } + + template + FieldDefPtr FindFieldByName(const T& str) const { + return FindFieldByName(str.c_str(), str.size()); + } + + OneofDefPtr FindOneofByName(const char* name, size_t len) const { + return OneofDefPtr(upb_msgdef_ntoo(ptr_, name, len)); + } + + OneofDefPtr FindOneofByName(const char *name) const { + return OneofDefPtr(upb_msgdef_ntooz(ptr_, name)); + } + + template + OneofDefPtr FindOneofByName(const T &str) const { + return FindOneofByName(str.c_str(), str.size()); + } + + /* Is this message a map entry? */ + bool mapentry() const { return upb_msgdef_mapentry(ptr_); } + + /* Return the type of well known type message. UPB_WELLKNOWN_UNSPECIFIED for + * non-well-known message. */ + upb_wellknowntype_t wellknowntype() const { + return upb_msgdef_wellknowntype(ptr_); + } + + /* Whether is a number wrapper. */ + bool isnumberwrapper() const { return upb_msgdef_isnumberwrapper(ptr_); } + + /* Iteration over fields. The order is undefined. */ + class const_field_iterator + : public std::iterator { + public: + void operator++() { upb_msg_field_next(&iter_); } + + FieldDefPtr operator*() const { + return FieldDefPtr(upb_msg_iter_field(&iter_)); + } + + bool operator!=(const const_field_iterator &other) const { + return !upb_msg_field_iter_isequal(&iter_, &other.iter_); + } + + bool operator==(const const_field_iterator &other) const { + return upb_msg_field_iter_isequal(&iter_, &other.iter_); + } + + private: + friend class MessageDefPtr; + + explicit const_field_iterator() {} + + explicit const_field_iterator(MessageDefPtr msg) { + upb_msg_field_begin(&iter_, msg.ptr()); + } + + static const_field_iterator end() { + const_field_iterator iter; + upb_msg_field_iter_setdone(&iter.iter_); + return iter; + } + + upb_msg_field_iter iter_; + }; + + /* Iteration over oneofs. The order is undefined. */ + class const_oneof_iterator + : public std::iterator { + public: + + void operator++() { upb_msg_oneof_next(&iter_); } + + OneofDefPtr operator*() const { + return OneofDefPtr(upb_msg_iter_oneof(&iter_)); + } + + bool operator!=(const const_oneof_iterator& other) const { + return !upb_msg_oneof_iter_isequal(&iter_, &other.iter_); + } + + bool operator==(const const_oneof_iterator &other) const { + return upb_msg_oneof_iter_isequal(&iter_, &other.iter_); + } + + private: + friend class MessageDefPtr; + + const_oneof_iterator() {} + + explicit const_oneof_iterator(MessageDefPtr msg) { + upb_msg_oneof_begin(&iter_, msg.ptr()); + } + + static const_oneof_iterator end() { + const_oneof_iterator iter; + upb_msg_oneof_iter_setdone(&iter.iter_); + return iter; + } + + upb_msg_oneof_iter iter_; + }; + + class ConstFieldAccessor { + public: + explicit ConstFieldAccessor(const upb_msgdef* md) : md_(md) {} + const_field_iterator begin() { return MessageDefPtr(md_).field_begin(); } + const_field_iterator end() { return MessageDefPtr(md_).field_end(); } + private: + const upb_msgdef* md_; + }; + + class ConstOneofAccessor { + public: + explicit ConstOneofAccessor(const upb_msgdef* md) : md_(md) {} + const_oneof_iterator begin() { return MessageDefPtr(md_).oneof_begin(); } + const_oneof_iterator end() { return MessageDefPtr(md_).oneof_end(); } + private: + const upb_msgdef* md_; + }; + + const_field_iterator field_begin() const { + return const_field_iterator(*this); + } + + const_field_iterator field_end() const { return const_field_iterator::end(); } + + const_oneof_iterator oneof_begin() const { + return const_oneof_iterator(*this); + } + + const_oneof_iterator oneof_end() const { return const_oneof_iterator::end(); } + + ConstFieldAccessor fields() const { return ConstFieldAccessor(ptr()); } + ConstOneofAccessor oneofs() const { return ConstOneofAccessor(ptr()); } + + private: + const upb_msgdef* ptr_; +}; + +inline upb::MessageDefPtr upb::FieldDefPtr::message_subdef() const { + return MessageDefPtr(upb_fielddef_msgsubdef(ptr_)); +} + +inline upb::MessageDefPtr upb::FieldDefPtr::containing_type() const { + return MessageDefPtr(upb_fielddef_containingtype(ptr_)); +} + +inline upb::MessageDefPtr upb::OneofDefPtr::containing_type() const { + return MessageDefPtr(upb_oneofdef_containingtype(ptr_)); +} + +#endif /* __cplusplus */ + +/* upb_enumdef ****************************************************************/ + +typedef upb_strtable_iter upb_enum_iter; + +const char *upb_enumdef_fullname(const upb_enumdef *e); +const char *upb_enumdef_name(const upb_enumdef *e); +const upb_filedef *upb_enumdef_file(const upb_enumdef *e); +int32_t upb_enumdef_default(const upb_enumdef *e); +int upb_enumdef_numvals(const upb_enumdef *e); + +/* Enum lookups: + * - ntoi: look up a name with specified length. + * - ntoiz: look up a name provided as a null-terminated string. + * - iton: look up an integer, returning the name as a null-terminated + * string. */ +bool upb_enumdef_ntoi(const upb_enumdef *e, const char *name, size_t len, + int32_t *num); +UPB_INLINE bool upb_enumdef_ntoiz(const upb_enumdef *e, + const char *name, int32_t *num) { + return upb_enumdef_ntoi(e, name, strlen(name), num); +} +const char *upb_enumdef_iton(const upb_enumdef *e, int32_t num); + +/* upb_enum_iter i; + * for(upb_enum_begin(&i, e); !upb_enum_done(&i); upb_enum_next(&i)) { + * // ... + * } + */ +void upb_enum_begin(upb_enum_iter *iter, const upb_enumdef *e); +void upb_enum_next(upb_enum_iter *iter); +bool upb_enum_done(upb_enum_iter *iter); +const char *upb_enum_iter_name(upb_enum_iter *iter); +int32_t upb_enum_iter_number(upb_enum_iter *iter); + +#ifdef __cplusplus + +class upb::EnumDefPtr { + public: + EnumDefPtr() : ptr_(nullptr) {} + explicit EnumDefPtr(const upb_enumdef* ptr) : ptr_(ptr) {} + + const upb_enumdef* ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + const char* full_name() const { return upb_enumdef_fullname(ptr_); } + const char* name() const { return upb_enumdef_name(ptr_); } + + /* The value that is used as the default when no field default is specified. + * If not set explicitly, the first value that was added will be used. + * The default value must be a member of the enum. + * Requires that value_count() > 0. */ + int32_t default_value() const { return upb_enumdef_default(ptr_); } + + /* Returns the number of values currently defined in the enum. Note that + * multiple names can refer to the same number, so this may be greater than + * the total number of unique numbers. */ + int value_count() const { return upb_enumdef_numvals(ptr_); } + + /* Lookups from name to integer, returning true if found. */ + bool FindValueByName(const char *name, int32_t *num) const { + return upb_enumdef_ntoiz(ptr_, name, num); + } + + /* Finds the name corresponding to the given number, or NULL if none was + * found. If more than one name corresponds to this number, returns the + * first one that was added. */ + const char *FindValueByNumber(int32_t num) const { + return upb_enumdef_iton(ptr_, num); + } + + /* Iteration over name/value pairs. The order is undefined. + * Adding an enum val invalidates any iterators. + * + * TODO: make compatible with range-for, with elements as pairs? */ + class Iterator { + public: + explicit Iterator(EnumDefPtr e) { upb_enum_begin(&iter_, e.ptr()); } + + int32_t number() { return upb_enum_iter_number(&iter_); } + const char *name() { return upb_enum_iter_name(&iter_); } + bool Done() { return upb_enum_done(&iter_); } + void Next() { return upb_enum_next(&iter_); } + + private: + upb_enum_iter iter_; + }; + + private: + const upb_enumdef *ptr_; +}; + +inline upb::EnumDefPtr upb::FieldDefPtr::enum_subdef() const { + return EnumDefPtr(upb_fielddef_enumsubdef(ptr_)); +} + +#endif /* __cplusplus */ + +/* upb_filedef ****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_filedef_name(const upb_filedef *f); +const char *upb_filedef_package(const upb_filedef *f); +const char *upb_filedef_phpprefix(const upb_filedef *f); +const char *upb_filedef_phpnamespace(const upb_filedef *f); +upb_syntax_t upb_filedef_syntax(const upb_filedef *f); +int upb_filedef_depcount(const upb_filedef *f); +int upb_filedef_msgcount(const upb_filedef *f); +int upb_filedef_enumcount(const upb_filedef *f); +const upb_filedef *upb_filedef_dep(const upb_filedef *f, int i); +const upb_msgdef *upb_filedef_msg(const upb_filedef *f, int i); +const upb_enumdef *upb_filedef_enum(const upb_filedef *f, int i); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Class that represents a .proto file with some things defined in it. + * + * Many users won't care about FileDefs, but they are necessary if you want to + * read the values of file-level options. */ +class upb::FileDefPtr { + public: + explicit FileDefPtr(const upb_filedef *ptr) : ptr_(ptr) {} + + const upb_filedef* ptr() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + + /* Get/set name of the file (eg. "foo/bar.proto"). */ + const char* name() const { return upb_filedef_name(ptr_); } + + /* Package name for definitions inside the file (eg. "foo.bar"). */ + const char* package() const { return upb_filedef_package(ptr_); } + + /* Sets the php class prefix which is prepended to all php generated classes + * from this .proto. Default is empty. */ + const char* phpprefix() const { return upb_filedef_phpprefix(ptr_); } + + /* Use this option to change the namespace of php generated classes. Default + * is empty. When this option is empty, the package name will be used for + * determining the namespace. */ + const char* phpnamespace() const { return upb_filedef_phpnamespace(ptr_); } + + /* Syntax for the file. Defaults to proto2. */ + upb_syntax_t syntax() const { return upb_filedef_syntax(ptr_); } + + /* Get the list of dependencies from the file. These are returned in the + * order that they were added to the FileDefPtr. */ + int dependency_count() const { return upb_filedef_depcount(ptr_); } + const FileDefPtr dependency(int index) const { + return FileDefPtr(upb_filedef_dep(ptr_, index)); + } + + private: + const upb_filedef* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_symtab *****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +upb_symtab *upb_symtab_new(void); +void upb_symtab_free(upb_symtab* s); +const upb_msgdef *upb_symtab_lookupmsg(const upb_symtab *s, const char *sym); +const upb_msgdef *upb_symtab_lookupmsg2( + const upb_symtab *s, const char *sym, size_t len); +const upb_enumdef *upb_symtab_lookupenum(const upb_symtab *s, const char *sym); +const upb_filedef *upb_symtab_lookupfile(const upb_symtab *s, const char *name); +int upb_symtab_filecount(const upb_symtab *s); +const upb_filedef *upb_symtab_addfile( + upb_symtab *s, const google_protobuf_FileDescriptorProto *file, + upb_status *status); + +/* For generated code only: loads a generated descriptor. */ +typedef struct upb_def_init { + struct upb_def_init **deps; + const char *filename; + upb_strview descriptor; +} upb_def_init; + +bool _upb_symtab_loaddefinit(upb_symtab *s, const upb_def_init *init); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Non-const methods in upb::SymbolTable are NOT thread-safe. */ +class upb::SymbolTable { + public: + SymbolTable() : ptr_(upb_symtab_new(), upb_symtab_free) {} + explicit SymbolTable(upb_symtab* s) : ptr_(s, upb_symtab_free) {} + + const upb_symtab* ptr() const { return ptr_.get(); } + upb_symtab* ptr() { return ptr_.get(); } + + /* Finds an entry in the symbol table with this exact name. If not found, + * returns NULL. */ + MessageDefPtr LookupMessage(const char *sym) const { + return MessageDefPtr(upb_symtab_lookupmsg(ptr_.get(), sym)); + } + + EnumDefPtr LookupEnum(const char *sym) const { + return EnumDefPtr(upb_symtab_lookupenum(ptr_.get(), sym)); + } + + FileDefPtr LookupFile(const char *name) const { + return FileDefPtr(upb_symtab_lookupfile(ptr_.get(), name)); + } + + /* TODO: iteration? */ + + /* Adds the given serialized FileDescriptorProto to the pool. */ + FileDefPtr AddFile(const google_protobuf_FileDescriptorProto *file_proto, + Status *status) { + return FileDefPtr( + upb_symtab_addfile(ptr_.get(), file_proto, status->ptr())); + } + + private: + std::unique_ptr ptr_; +}; + +UPB_INLINE const char* upb_safecstr(const std::string& str) { + UPB_ASSERT(str.size() == std::strlen(str.c_str())); + return str.c_str(); +} + +#endif /* __cplusplus */ + +#include "upb/port_undef.inc" + +#endif /* UPB_DEF_H_ */ diff --git a/upb/encode.c b/upb/encode.c new file mode 100644 index 00000000000..43d24cdbfcc --- /dev/null +++ b/upb/encode.c @@ -0,0 +1,378 @@ +/* We encode backwards, to avoid pre-computing lengths (one-pass encode). */ + +#include "upb/encode.h" + +#include + +#include "upb/msg.h" +#include "upb/upb.h" + +#include "upb/port_def.inc" + +#define UPB_PB_VARINT_MAX_LEN 10 +#define CHK(x) do { if (!(x)) { return false; } } while(0) + +static size_t upb_encode_varint(uint64_t val, char *buf) { + size_t i; + if (val < 128) { buf[0] = val; return 1; } + i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +static uint32_t upb_zzencode_32(int32_t n) { return ((uint32_t)n << 1) ^ (n >> 31); } +static uint64_t upb_zzencode_64(int64_t n) { return ((uint64_t)n << 1) ^ (n >> 63); } + +typedef struct { + upb_alloc *alloc; + char *buf, *ptr, *limit; +} upb_encstate; + +static size_t upb_roundup_pow2(size_t bytes) { + size_t ret = 128; + while (ret < bytes) { + ret *= 2; + } + return ret; +} + +static bool upb_encode_growbuffer(upb_encstate *e, size_t bytes) { + size_t old_size = e->limit - e->buf; + size_t new_size = upb_roundup_pow2(bytes + (e->limit - e->ptr)); + char *new_buf = upb_realloc(e->alloc, e->buf, old_size, new_size); + CHK(new_buf); + + /* We want previous data at the end, realloc() put it at the beginning. */ + if (old_size > 0) { + memmove(new_buf + new_size - old_size, e->buf, old_size); + } + + e->ptr = new_buf + new_size - (e->limit - e->ptr); + e->limit = new_buf + new_size; + e->buf = new_buf; + return true; +} + +/* Call to ensure that at least "bytes" bytes are available for writing at + * e->ptr. Returns false if the bytes could not be allocated. */ +static bool upb_encode_reserve(upb_encstate *e, size_t bytes) { + CHK(UPB_LIKELY((size_t)(e->ptr - e->buf) >= bytes) || + upb_encode_growbuffer(e, bytes)); + + e->ptr -= bytes; + return true; +} + +/* Writes the given bytes to the buffer, handling reserve/advance. */ +static bool upb_put_bytes(upb_encstate *e, const void *data, size_t len) { + CHK(upb_encode_reserve(e, len)); + memcpy(e->ptr, data, len); + return true; +} + +static bool upb_put_fixed64(upb_encstate *e, uint64_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return upb_put_bytes(e, &val, sizeof(uint64_t)); +} + +static bool upb_put_fixed32(upb_encstate *e, uint32_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return upb_put_bytes(e, &val, sizeof(uint32_t)); +} + +static bool upb_put_varint(upb_encstate *e, uint64_t val) { + size_t len; + char *start; + CHK(upb_encode_reserve(e, UPB_PB_VARINT_MAX_LEN)); + len = upb_encode_varint(val, e->ptr); + start = e->ptr + UPB_PB_VARINT_MAX_LEN - len; + memmove(start, e->ptr, len); + e->ptr = start; + return true; +} + +static bool upb_put_double(upb_encstate *e, double d) { + uint64_t u64; + UPB_ASSERT(sizeof(double) == sizeof(uint64_t)); + memcpy(&u64, &d, sizeof(uint64_t)); + return upb_put_fixed64(e, u64); +} + +static bool upb_put_float(upb_encstate *e, float d) { + uint32_t u32; + UPB_ASSERT(sizeof(float) == sizeof(uint32_t)); + memcpy(&u32, &d, sizeof(uint32_t)); + return upb_put_fixed32(e, u32); +} + +static uint32_t upb_readcase(const char *msg, const upb_msglayout_field *f) { + uint32_t ret; + uint32_t offset = ~f->presence; + memcpy(&ret, msg + offset, sizeof(ret)); + return ret; +} + +static bool upb_readhasbit(const char *msg, const upb_msglayout_field *f) { + uint32_t hasbit = f->presence; + UPB_ASSERT(f->presence > 0); + return msg[hasbit / 8] & (1 << (hasbit % 8)); +} + +static bool upb_put_tag(upb_encstate *e, int field_number, int wire_type) { + return upb_put_varint(e, (field_number << 3) | wire_type); +} + +static bool upb_put_fixedarray(upb_encstate *e, const upb_array *arr, + size_t size) { + size_t bytes = arr->len * size; + return upb_put_bytes(e, arr->data, bytes) && upb_put_varint(e, bytes); +} + +bool upb_encode_message(upb_encstate *e, const char *msg, + const upb_msglayout *m, size_t *size); + +static bool upb_encode_array(upb_encstate *e, const char *field_mem, + const upb_msglayout *m, + const upb_msglayout_field *f) { + const upb_array *arr = *(const upb_array**)field_mem; + + if (arr == NULL || arr->len == 0) { + return true; + } + +#define VARINT_CASE(ctype, encode) { \ + ctype *start = arr->data; \ + ctype *ptr = start + arr->len; \ + size_t pre_len = e->limit - e->ptr; \ + do { \ + ptr--; \ + CHK(upb_put_varint(e, encode)); \ + } while (ptr != start); \ + CHK(upb_put_varint(e, e->limit - e->ptr - pre_len)); \ +} \ +break; \ +do { ; } while(0) + + switch (f->descriptortype) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + CHK(upb_put_fixedarray(e, arr, sizeof(double))); + break; + case UPB_DESCRIPTOR_TYPE_FLOAT: + CHK(upb_put_fixedarray(e, arr, sizeof(float))); + break; + case UPB_DESCRIPTOR_TYPE_SFIXED64: + case UPB_DESCRIPTOR_TYPE_FIXED64: + CHK(upb_put_fixedarray(e, arr, sizeof(uint64_t))); + break; + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + CHK(upb_put_fixedarray(e, arr, sizeof(uint32_t))); + break; + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + VARINT_CASE(uint64_t, *ptr); + case UPB_DESCRIPTOR_TYPE_UINT32: + VARINT_CASE(uint32_t, *ptr); + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_ENUM: + VARINT_CASE(int32_t, (int64_t)*ptr); + case UPB_DESCRIPTOR_TYPE_BOOL: + VARINT_CASE(bool, *ptr); + case UPB_DESCRIPTOR_TYPE_SINT32: + VARINT_CASE(int32_t, upb_zzencode_32(*ptr)); + case UPB_DESCRIPTOR_TYPE_SINT64: + VARINT_CASE(int64_t, upb_zzencode_64(*ptr)); + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview *start = arr->data; + upb_strview *ptr = start + arr->len; + do { + ptr--; + CHK(upb_put_bytes(e, ptr->data, ptr->size) && + upb_put_varint(e, ptr->size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED)); + } while (ptr != start); + return true; + } + case UPB_DESCRIPTOR_TYPE_GROUP: { + void **start = arr->data; + void **ptr = start + arr->len; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + do { + size_t size; + ptr--; + CHK(upb_put_tag(e, f->number, UPB_WIRE_TYPE_END_GROUP) && + upb_encode_message(e, *ptr, subm, &size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_START_GROUP)); + } while (ptr != start); + return true; + } + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + void **start = arr->data; + void **ptr = start + arr->len; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + do { + size_t size; + ptr--; + CHK(upb_encode_message(e, *ptr, subm, &size) && + upb_put_varint(e, size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED)); + } while (ptr != start); + return true; + } + } +#undef VARINT_CASE + + /* We encode all primitive arrays as packed, regardless of what was specified + * in the .proto file. Could special case 1-sized arrays. */ + CHK(upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED)); + return true; +} + +static bool upb_encode_scalarfield(upb_encstate *e, const char *field_mem, + const upb_msglayout *m, + const upb_msglayout_field *f, + bool skip_zero_value) { +#define CASE(ctype, type, wire_type, encodeval) do { \ + ctype val = *(ctype*)field_mem; \ + if (skip_zero_value && val == 0) { \ + return true; \ + } \ + return upb_put_ ## type(e, encodeval) && \ + upb_put_tag(e, f->number, wire_type); \ +} while(0) + + switch (f->descriptortype) { + case UPB_DESCRIPTOR_TYPE_DOUBLE: + CASE(double, double, UPB_WIRE_TYPE_64BIT, val); + case UPB_DESCRIPTOR_TYPE_FLOAT: + CASE(float, float, UPB_WIRE_TYPE_32BIT, val); + case UPB_DESCRIPTOR_TYPE_INT64: + case UPB_DESCRIPTOR_TYPE_UINT64: + CASE(uint64_t, varint, UPB_WIRE_TYPE_VARINT, val); + case UPB_DESCRIPTOR_TYPE_UINT32: + CASE(uint32_t, varint, UPB_WIRE_TYPE_VARINT, val); + case UPB_DESCRIPTOR_TYPE_INT32: + case UPB_DESCRIPTOR_TYPE_ENUM: + CASE(int32_t, varint, UPB_WIRE_TYPE_VARINT, (int64_t)val); + case UPB_DESCRIPTOR_TYPE_SFIXED64: + case UPB_DESCRIPTOR_TYPE_FIXED64: + CASE(uint64_t, fixed64, UPB_WIRE_TYPE_64BIT, val); + case UPB_DESCRIPTOR_TYPE_FIXED32: + case UPB_DESCRIPTOR_TYPE_SFIXED32: + CASE(uint32_t, fixed32, UPB_WIRE_TYPE_32BIT, val); + case UPB_DESCRIPTOR_TYPE_BOOL: + CASE(bool, varint, UPB_WIRE_TYPE_VARINT, val); + case UPB_DESCRIPTOR_TYPE_SINT32: + CASE(int32_t, varint, UPB_WIRE_TYPE_VARINT, upb_zzencode_32(val)); + case UPB_DESCRIPTOR_TYPE_SINT64: + CASE(int64_t, varint, UPB_WIRE_TYPE_VARINT, upb_zzencode_64(val)); + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: { + upb_strview view = *(upb_strview*)field_mem; + if (skip_zero_value && view.size == 0) { + return true; + } + return upb_put_bytes(e, view.data, view.size) && + upb_put_varint(e, view.size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED); + } + case UPB_DESCRIPTOR_TYPE_GROUP: { + size_t size; + void *submsg = *(void **)field_mem; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + if (submsg == NULL) { + return true; + } + return upb_put_tag(e, f->number, UPB_WIRE_TYPE_END_GROUP) && + upb_encode_message(e, submsg, subm, &size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_START_GROUP); + } + case UPB_DESCRIPTOR_TYPE_MESSAGE: { + size_t size; + void *submsg = *(void **)field_mem; + const upb_msglayout *subm = m->submsgs[f->submsg_index]; + if (submsg == NULL) { + return true; + } + return upb_encode_message(e, submsg, subm, &size) && + upb_put_varint(e, size) && + upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED); + } + } +#undef CASE + UPB_UNREACHABLE(); +} + +bool upb_encode_message(upb_encstate *e, const char *msg, + const upb_msglayout *m, size_t *size) { + int i; + size_t pre_len = e->limit - e->ptr; + const char *unknown; + size_t unknown_size; + + for (i = m->field_count - 1; i >= 0; i--) { + const upb_msglayout_field *f = &m->fields[i]; + + if (f->label == UPB_LABEL_REPEATED) { + CHK(upb_encode_array(e, msg + f->offset, m, f)); + } else { + bool skip_empty = false; + if (f->presence == 0) { + /* Proto3 presence. */ + skip_empty = true; + } else if (f->presence > 0) { + /* Proto2 presence: hasbit. */ + if (!upb_readhasbit(msg, f)) { + continue; + } + } else { + /* Field is in a oneof. */ + if (upb_readcase(msg, f) != f->number) { + continue; + } + } + CHK(upb_encode_scalarfield(e, msg + f->offset, m, f, skip_empty)); + } + } + + unknown = upb_msg_getunknown(msg, &unknown_size); + + if (unknown) { + upb_put_bytes(e, unknown, unknown_size); + } + + *size = (e->limit - e->ptr) - pre_len; + return true; +} + +char *upb_encode(const void *msg, const upb_msglayout *m, upb_arena *arena, + size_t *size) { + upb_encstate e; + e.alloc = upb_arena_alloc(arena); + e.buf = NULL; + e.limit = NULL; + e.ptr = NULL; + + if (!upb_encode_message(&e, msg, m, size)) { + *size = 0; + return NULL; + } + + *size = e.limit - e.ptr; + + if (*size == 0) { + static char ch; + return &ch; + } else { + UPB_ASSERT(e.ptr); + return e.ptr; + } +} + +#undef CHK diff --git a/upb/encode.h b/upb/encode.h new file mode 100644 index 00000000000..68427770585 --- /dev/null +++ b/upb/encode.h @@ -0,0 +1,21 @@ +/* +** upb_encode: parsing into a upb_msg using a upb_msglayout. +*/ + +#ifndef UPB_ENCODE_H_ +#define UPB_ENCODE_H_ + +#include "upb/msg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +char *upb_encode(const void *msg, const upb_msglayout *l, upb_arena *arena, + size_t *size); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_ENCODE_H_ */ diff --git a/upb/generated_util.h b/upb/generated_util.h new file mode 100644 index 00000000000..234bcdad3c9 --- /dev/null +++ b/upb/generated_util.h @@ -0,0 +1,105 @@ +/* +** Functions for use by generated code. These are not public and users must +** not call them directly. +*/ + +#ifndef UPB_GENERATED_UTIL_H_ +#define UPB_GENERATED_UTIL_H_ + +#include +#include "upb/msg.h" + +#include "upb/port_def.inc" + +#define PTR_AT(msg, ofs, type) (type*)((const char*)msg + ofs) + +UPB_INLINE const void *_upb_array_accessor(const void *msg, size_t ofs, + size_t *size) { + const upb_array *arr = *PTR_AT(msg, ofs, const upb_array*); + if (arr) { + if (size) *size = arr->len; + return arr->data; + } else { + if (size) *size = 0; + return NULL; + } +} + +UPB_INLINE void *_upb_array_mutable_accessor(void *msg, size_t ofs, + size_t *size) { + upb_array *arr = *PTR_AT(msg, ofs, upb_array*); + if (arr) { + if (size) *size = arr->len; + return arr->data; + } else { + if (size) *size = 0; + return NULL; + } +} + +/* TODO(haberman): this is a mess. It will improve when upb_array no longer + * carries reflective state (type, elem_size). */ +UPB_INLINE void *_upb_array_resize_accessor(void *msg, size_t ofs, size_t size, + size_t elem_size, + upb_fieldtype_t type, + upb_arena *arena) { + upb_array *arr = *PTR_AT(msg, ofs, upb_array*); + + if (!arr) { + arr = upb_array_new(arena); + if (!arr) return NULL; + *PTR_AT(msg, ofs, upb_array*) = arr; + } + + if (size > arr->size) { + size_t new_size = UPB_MAX(arr->size, 4); + size_t old_bytes = arr->size * elem_size; + size_t new_bytes; + while (new_size < size) new_size *= 2; + new_bytes = new_size * elem_size; + arr->data = upb_arena_realloc(arena, arr->data, old_bytes, new_bytes); + if (!arr->data) { + return NULL; + } + arr->size = new_size; + } + + arr->len = size; + return arr->data; +} + +UPB_INLINE bool _upb_array_append_accessor(void *msg, size_t ofs, + size_t elem_size, + upb_fieldtype_t type, + const void *value, + upb_arena *arena) { + upb_array *arr = *PTR_AT(msg, ofs, upb_array*); + size_t i = arr ? arr->len : 0; + void *data = + _upb_array_resize_accessor(msg, ofs, i + 1, elem_size, type, arena); + if (!data) return false; + memcpy(PTR_AT(data, i * elem_size, char), value, elem_size); + return true; +} + +UPB_INLINE bool _upb_has_field(const void *msg, size_t idx) { + return (*PTR_AT(msg, idx / 8, const char) & (1 << (idx % 8))) != 0; +} + +UPB_INLINE bool _upb_sethas(const void *msg, size_t idx) { + return (*PTR_AT(msg, idx / 8, char)) |= (char)(1 << (idx % 8)); +} + +UPB_INLINE bool _upb_clearhas(const void *msg, size_t idx) { + return (*PTR_AT(msg, idx / 8, char)) &= (char)(~(1 << (idx % 8))); +} + +UPB_INLINE bool _upb_has_oneof_field(const void *msg, size_t case_ofs, int32_t num) { + return *PTR_AT(msg, case_ofs, int32_t) == num; +} + +#undef PTR_AT + +#include "upb/port_undef.inc" + +#endif /* UPB_GENERATED_UTIL_H_ */ diff --git a/upb/handlers-inl.h b/upb/handlers-inl.h new file mode 100644 index 00000000000..8f8634bfaa0 --- /dev/null +++ b/upb/handlers-inl.h @@ -0,0 +1,923 @@ +/* +** Inline definitions for handlers.h, which are particularly long and a bit +** tricky. +*/ + +#ifndef UPB_HANDLERS_INL_H_ +#define UPB_HANDLERS_INL_H_ + +#include +#include +#include "upb/handlers.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus + +/* Type detection and typedefs for integer types. + * For platforms where there are multiple 32-bit or 64-bit types, we need to be + * able to enumerate them so we can properly create overloads for all variants. + * + * If any platform existed where there were three integer types with the same + * size, this would have to become more complicated. For example, short, int, + * and long could all be 32-bits. Even more diabolically, short, int, long, + * and long long could all be 64 bits and still be standard-compliant. + * However, few platforms are this strange, and it's unlikely that upb will be + * used on the strangest ones. */ + +/* Can't count on stdint.h limits like INT32_MAX, because in C++ these are + * only defined when __STDC_LIMIT_MACROS are defined before the *first* include + * of stdint.h. We can't guarantee that someone else didn't include these first + * without defining __STDC_LIMIT_MACROS. */ +#define UPB_INT32_MAX 0x7fffffffLL +#define UPB_INT32_MIN (-UPB_INT32_MAX - 1) +#define UPB_INT64_MAX 0x7fffffffffffffffLL +#define UPB_INT64_MIN (-UPB_INT64_MAX - 1) + +#if INT_MAX == UPB_INT32_MAX && INT_MIN == UPB_INT32_MIN +#define UPB_INT_IS_32BITS 1 +#endif + +#if LONG_MAX == UPB_INT32_MAX && LONG_MIN == UPB_INT32_MIN +#define UPB_LONG_IS_32BITS 1 +#endif + +#if LONG_MAX == UPB_INT64_MAX && LONG_MIN == UPB_INT64_MIN +#define UPB_LONG_IS_64BITS 1 +#endif + +#if LLONG_MAX == UPB_INT64_MAX && LLONG_MIN == UPB_INT64_MIN +#define UPB_LLONG_IS_64BITS 1 +#endif + +/* We use macros instead of typedefs so we can undefine them later and avoid + * leaking them outside this header file. */ +#if UPB_INT_IS_32BITS +#define UPB_INT32_T int +#define UPB_UINT32_T unsigned int + +#if UPB_LONG_IS_32BITS +#define UPB_TWO_32BIT_TYPES 1 +#define UPB_INT32ALT_T long +#define UPB_UINT32ALT_T unsigned long +#endif /* UPB_LONG_IS_32BITS */ + +#elif UPB_LONG_IS_32BITS /* && !UPB_INT_IS_32BITS */ +#define UPB_INT32_T long +#define UPB_UINT32_T unsigned long +#endif /* UPB_INT_IS_32BITS */ + + +#if UPB_LONG_IS_64BITS +#define UPB_INT64_T long +#define UPB_UINT64_T unsigned long + +#if UPB_LLONG_IS_64BITS +#define UPB_TWO_64BIT_TYPES 1 +#define UPB_INT64ALT_T long long +#define UPB_UINT64ALT_T unsigned long long +#endif /* UPB_LLONG_IS_64BITS */ + +#elif UPB_LLONG_IS_64BITS /* && !UPB_LONG_IS_64BITS */ +#define UPB_INT64_T long long +#define UPB_UINT64_T unsigned long long +#endif /* UPB_LONG_IS_64BITS */ + +#undef UPB_INT32_MAX +#undef UPB_INT32_MIN +#undef UPB_INT64_MAX +#undef UPB_INT64_MIN +#undef UPB_INT_IS_32BITS +#undef UPB_LONG_IS_32BITS +#undef UPB_LONG_IS_64BITS +#undef UPB_LLONG_IS_64BITS + + +namespace upb { + +typedef void CleanupFunc(void *ptr); + +/* Template to remove "const" from "const T*" and just return "T*". + * + * We define a nonsense default because otherwise it will fail to instantiate as + * a function parameter type even in cases where we don't expect any caller to + * actually match the overload. */ +class CouldntRemoveConst {}; +template struct remove_constptr { typedef CouldntRemoveConst type; }; +template struct remove_constptr { typedef T *type; }; + +/* Template that we use below to remove a template specialization from + * consideration if it matches a specific type. */ +template struct disable_if_same { typedef void Type; }; +template struct disable_if_same {}; + +template void DeletePointer(void *p) { delete static_cast(p); } + +template +struct FirstUnlessVoidOrBool { + typedef T1 value; +}; + +template +struct FirstUnlessVoidOrBool { + typedef T2 value; +}; + +template +struct FirstUnlessVoidOrBool { + typedef T2 value; +}; + +template +struct is_same { + static bool value; +}; + +template +struct is_same { + static bool value; +}; + +template +bool is_same::value = false; + +template +bool is_same::value = true; + +/* FuncInfo *******************************************************************/ + +/* Info about the user's original, pre-wrapped function. */ +template +struct FuncInfo { + /* The type of the closure that the function takes (its first param). */ + typedef C Closure; + + /* The return type. */ + typedef R Return; +}; + +/* Func ***********************************************************************/ + +/* Func1, Func2, Func3: Template classes representing a function and its + * signature. + * + * Since the function is a template parameter, calling the function can be + * inlined at compile-time and does not require a function pointer at runtime. + * These functions are not bound to a handler data so have no data or cleanup + * handler. */ +struct UnboundFunc { + CleanupFunc *GetCleanup() { return nullptr; } + void *GetData() { return nullptr; } +}; + +template +struct Func1 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1) { return F(p1); } +}; + +template +struct Func2 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2) { return F(p1, p2); } +}; + +template +struct Func3 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2, P3 p3) { return F(p1, p2, p3); } +}; + +template +struct Func4 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2, P3 p3, P4 p4) { return F(p1, p2, p3, p4); } +}; + +template +struct Func5 : public UnboundFunc { + typedef R Return; + typedef I FuncInfo; + static R Call(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) { + return F(p1, p2, p3, p4, p5); + } +}; + +/* BoundFunc ******************************************************************/ + +/* BoundFunc2, BoundFunc3: Like Func2/Func3 except also contains a value that + * shall be bound to the function's second parameter. + * + * Note that the second parameter is a const pointer, but our stored bound value + * is non-const so we can free it when the handlers are destroyed. */ +template +struct BoundFunc { + typedef typename remove_constptr::type MutableP2; + explicit BoundFunc(MutableP2 data_) : data(data_) {} + CleanupFunc *GetCleanup() { return &DeletePointer; } + MutableP2 GetData() { return data; } + MutableP2 data; +}; + +template +struct BoundFunc2 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc2(typename Base::MutableP2 arg) : Base(arg) {} +}; + +template +struct BoundFunc3 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc3(typename Base::MutableP2 arg) : Base(arg) {} +}; + +template +struct BoundFunc4 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc4(typename Base::MutableP2 arg) : Base(arg) {} +}; + +template +struct BoundFunc5 : public BoundFunc { + typedef BoundFunc Base; + typedef I FuncInfo; + explicit BoundFunc5(typename Base::MutableP2 arg) : Base(arg) {} +}; + +/* FuncSig ********************************************************************/ + +/* FuncSig1, FuncSig2, FuncSig3: template classes reflecting a function + * *signature*, but without a specific function attached. + * + * These classes contain member functions that can be invoked with a + * specific function to return a Func/BoundFunc class. */ +template +struct FuncSig1 { + template + Func1 > GetFunc() { + return Func1 >(); + } +}; + +template +struct FuncSig2 { + template + Func2 > GetFunc() { + return Func2 >(); + } + + template + BoundFunc2 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc2 >(param2); + } +}; + +template +struct FuncSig3 { + template + Func3 > GetFunc() { + return Func3 >(); + } + + template + BoundFunc3 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc3 >(param2); + } +}; + +template +struct FuncSig4 { + template + Func4 > GetFunc() { + return Func4 >(); + } + + template + BoundFunc4 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc4 >(param2); + } +}; + +template +struct FuncSig5 { + template + Func5 > GetFunc() { + return Func5 >(); + } + + template + BoundFunc5 > GetFunc( + typename remove_constptr::type param2) { + return BoundFunc5 >(param2); + } +}; + +/* Overloaded template function that can construct the appropriate FuncSig* + * class given a function pointer by deducing the template parameters. */ +template +inline FuncSig1 MatchFunc(R (*f)(P1)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig1(); +} + +template +inline FuncSig2 MatchFunc(R (*f)(P1, P2)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig2(); +} + +template +inline FuncSig3 MatchFunc(R (*f)(P1, P2, P3)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig3(); +} + +template +inline FuncSig4 MatchFunc(R (*f)(P1, P2, P3, P4)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig4(); +} + +template +inline FuncSig5 MatchFunc(R (*f)(P1, P2, P3, P4, P5)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return FuncSig5(); +} + +/* MethodSig ******************************************************************/ + +/* CallMethod*: a function template that calls a given method. */ +template +R CallMethod0(C *obj) { + return ((*obj).*F)(); +} + +template +R CallMethod1(C *obj, P1 arg1) { + return ((*obj).*F)(arg1); +} + +template +R CallMethod2(C *obj, P1 arg1, P2 arg2) { + return ((*obj).*F)(arg1, arg2); +} + +template +R CallMethod3(C *obj, P1 arg1, P2 arg2, P3 arg3) { + return ((*obj).*F)(arg1, arg2, arg3); +} + +template +R CallMethod4(C *obj, P1 arg1, P2 arg2, P3 arg3, P4 arg4) { + return ((*obj).*F)(arg1, arg2, arg3, arg4); +} + +/* MethodSig: like FuncSig, but for member functions. + * + * GetFunc() returns a normal FuncN object, so after calling GetFunc() no + * more logic is required to special-case methods. */ +template +struct MethodSig0 { + template + Func1, FuncInfo > GetFunc() { + return Func1, FuncInfo >(); + } +}; + +template +struct MethodSig1 { + template + Func2, FuncInfo > GetFunc() { + return Func2, FuncInfo >(); + } + + template + BoundFunc2, FuncInfo > GetFunc( + typename remove_constptr::type param1) { + return BoundFunc2, FuncInfo >( + param1); + } +}; + +template +struct MethodSig2 { + template + Func3, FuncInfo > + GetFunc() { + return Func3, + FuncInfo >(); + } + + template + BoundFunc3, FuncInfo > + GetFunc(typename remove_constptr::type param1) { + return BoundFunc3, + FuncInfo >(param1); + } +}; + +template +struct MethodSig3 { + template + Func4, FuncInfo > + GetFunc() { + return Func4, + FuncInfo >(); + } + + template + BoundFunc4, + FuncInfo > + GetFunc(typename remove_constptr::type param1) { + return BoundFunc4, + FuncInfo >(param1); + } +}; + +template +struct MethodSig4 { + template + Func5, + FuncInfo > + GetFunc() { + return Func5, + FuncInfo >(); + } + + template + BoundFunc5, + FuncInfo > + GetFunc(typename remove_constptr::type param1) { + return BoundFunc5, FuncInfo >( + param1); + } +}; + +template +inline MethodSig0 MatchFunc(R (C::*f)()) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig0(); +} + +template +inline MethodSig1 MatchFunc(R (C::*f)(P1)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig1(); +} + +template +inline MethodSig2 MatchFunc(R (C::*f)(P1, P2)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig2(); +} + +template +inline MethodSig3 MatchFunc(R (C::*f)(P1, P2, P3)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig3(); +} + +template +inline MethodSig4 MatchFunc(R (C::*f)(P1, P2, P3, P4)) { + UPB_UNUSED(f); /* Only used for template parameter deduction. */ + return MethodSig4(); +} + +/* MaybeWrapReturn ************************************************************/ + +/* Template class that attempts to wrap the return value of the function so it + * matches the expected type. There are two main adjustments it may make: + * + * 1. If the function returns void, make it return the expected type and with + * a value that always indicates success. + * 2. If the function returns bool, make it return the expected type with a + * value that indicates success or failure. + * + * The "expected type" for return is: + * 1. void* for start handlers. If the closure parameter has a different type + * we will cast it to void* for the return in the success case. + * 2. size_t for string buffer handlers. + * 3. bool for everything else. */ + +/* Template parameters are FuncN type and desired return type. */ +template +struct MaybeWrapReturn; + +/* If the return type matches, return the given function unwrapped. */ +template +struct MaybeWrapReturn { + typedef F Func; +}; + +/* Function wrapper that munges the return value from void to (bool)true. */ +template +bool ReturnTrue2(P1 p1, P2 p2) { + F(p1, p2); + return true; +} + +template +bool ReturnTrue3(P1 p1, P2 p2, P3 p3) { + F(p1, p2, p3); + return true; +} + +/* Function wrapper that munges the return value from void to (void*)arg1 */ +template +void *ReturnClosure2(P1 p1, P2 p2) { + F(p1, p2); + return p1; +} + +template +void *ReturnClosure3(P1 p1, P2 p2, P3 p3) { + F(p1, p2, p3); + return p1; +} + +/* Function wrapper that munges the return value from R to void*. */ +template +void *CastReturnToVoidPtr2(P1 p1, P2 p2) { + return F(p1, p2); +} + +template +void *CastReturnToVoidPtr3(P1 p1, P2 p2, P3 p3) { + return F(p1, p2, p3); +} + +/* Function wrapper that munges the return value from bool to void*. */ +template +void *ReturnClosureOrBreak2(P1 p1, P2 p2) { + return F(p1, p2) ? p1 : UPB_BREAK; +} + +template +void *ReturnClosureOrBreak3(P1 p1, P2 p2, P3 p3) { + return F(p1, p2, p3) ? p1 : UPB_BREAK; +} + +/* For the string callback, which takes five params, returns the size param. */ +template +size_t ReturnStringLen(P1 p1, P2 p2, const char *p3, size_t p4, + const upb_bufhandle *p5) { + F(p1, p2, p3, p4, p5); + return p4; +} + +/* For the string callback, which takes five params, returns the size param or + * zero. */ +template +size_t ReturnNOr0(P1 p1, P2 p2, const char *p3, size_t p4, + const upb_bufhandle *p5) { + return F(p1, p2, p3, p4, p5) ? p4 : 0; +} + +/* If we have a function returning void but want a function returning bool, wrap + * it in a function that returns true. */ +template +struct MaybeWrapReturn, bool> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, bool> { + typedef Func3, I> Func; +}; + +/* If our function returns void but we want one returning void*, wrap it in a + * function that returns the first argument. */ +template +struct MaybeWrapReturn, void *> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, void *> { + typedef Func3, I> Func; +}; + +/* If our function returns R* but we want one returning void*, wrap it in a + * function that casts to void*. */ +template +struct MaybeWrapReturn, void *, + typename disable_if_same::Type> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, void *, + typename disable_if_same::Type> { + typedef Func3, I> + Func; +}; + +/* If our function returns bool but we want one returning void*, wrap it in a + * function that returns either the first param or UPB_BREAK. */ +template +struct MaybeWrapReturn, void *> { + typedef Func2, I> Func; +}; + +template +struct MaybeWrapReturn, void *> { + typedef Func3, I> + Func; +}; + +/* If our function returns void but we want one returning size_t, wrap it in a + * function that returns the size argument. */ +template +struct MaybeWrapReturn< + Func5, + size_t> { + typedef Func5, I> Func; +}; + +/* If our function returns bool but we want one returning size_t, wrap it in a + * function that returns either 0 or the buf size. */ +template +struct MaybeWrapReturn< + Func5, + size_t> { + typedef Func5, I> Func; +}; + +/* ConvertParams **************************************************************/ + +/* Template class that converts the function parameters if necessary, and + * ignores the HandlerData parameter if appropriate. + * + * Template parameter is the are FuncN function type. */ +template +struct ConvertParams; + +/* Function that discards the handler data parameter. */ +template +R IgnoreHandlerData2(void *p1, const void *hd) { + UPB_UNUSED(hd); + return F(static_cast(p1)); +} + +template +R IgnoreHandlerData3(void *p1, const void *hd, P2Wrapper p2) { + UPB_UNUSED(hd); + return F(static_cast(p1), p2); +} + +template +R IgnoreHandlerData4(void *p1, const void *hd, P2 p2, P3 p3) { + UPB_UNUSED(hd); + return F(static_cast(p1), p2, p3); +} + +template +R IgnoreHandlerData5(void *p1, const void *hd, P2 p2, P3 p3, P4 p4) { + UPB_UNUSED(hd); + return F(static_cast(p1), p2, p3, p4); +} + +template +R IgnoreHandlerDataIgnoreHandle(void *p1, const void *hd, const char *p2, + size_t p3, const upb_bufhandle *handle) { + UPB_UNUSED(hd); + UPB_UNUSED(handle); + return F(static_cast(p1), p2, p3); +} + +/* Function that casts the handler data parameter. */ +template +R CastHandlerData2(void *c, const void *hd) { + return F(static_cast(c), static_cast(hd)); +} + +template +R CastHandlerData3(void *c, const void *hd, P3Wrapper p3) { + return F(static_cast(c), static_cast(hd), p3); +} + +template +R CastHandlerData5(void *c, const void *hd, P3 p3, P4 p4, P5 p5) { + return F(static_cast(c), static_cast(hd), p3, p4, p5); +} + +template +R CastHandlerDataIgnoreHandle(void *c, const void *hd, const char *p3, + size_t p4, const upb_bufhandle *handle) { + UPB_UNUSED(handle); + return F(static_cast(c), static_cast(hd), p3, p4); +} + +/* For unbound functions, ignore the handler data. */ +template +struct ConvertParams, T> { + typedef Func2, I> Func; +}; + +template +struct ConvertParams, + R2 (*)(P1_2, P2_2, P3_2)> { + typedef Func3, I> Func; +}; + +/* For StringBuffer only; this ignores both the handler data and the + * upb_bufhandle. */ +template +struct ConvertParams, T> { + typedef Func5, + I> Func; +}; + +template +struct ConvertParams, T> { + typedef Func5, I> Func; +}; + +/* For bound functions, cast the handler data. */ +template +struct ConvertParams, T> { + typedef Func2, I> + Func; +}; + +template +struct ConvertParams, + R2 (*)(P1_2, P2_2, P3_2)> { + typedef Func3, I> Func; +}; + +/* For StringBuffer only; this ignores the upb_bufhandle. */ +template +struct ConvertParams, T> { + typedef Func5, I> + Func; +}; + +template +struct ConvertParams, T> { + typedef Func5, I> Func; +}; + +/* utype/ltype are upper/lower-case, ctype is canonical C type, vtype is + * variant C type. */ +#define TYPE_METHODS(utype, ltype, ctype, vtype) \ + template <> \ + struct CanonicalType { \ + typedef ctype Type; \ + }; \ + template <> \ + inline bool HandlersPtr::SetValueHandler( \ + FieldDefPtr f, const HandlersPtr::utype##Handler &handler) { \ + handler.AddCleanup(ptr()); \ + return upb_handlers_set##ltype(ptr(), f.ptr(), handler.handler(), \ + &handler.attr()); \ + } + +TYPE_METHODS(Double, double, double, double) +TYPE_METHODS(Float, float, float, float) +TYPE_METHODS(UInt64, uint64, uint64_t, UPB_UINT64_T) +TYPE_METHODS(UInt32, uint32, uint32_t, UPB_UINT32_T) +TYPE_METHODS(Int64, int64, int64_t, UPB_INT64_T) +TYPE_METHODS(Int32, int32, int32_t, UPB_INT32_T) +TYPE_METHODS(Bool, bool, bool, bool) + +#ifdef UPB_TWO_32BIT_TYPES +TYPE_METHODS(Int32, int32, int32_t, UPB_INT32ALT_T) +TYPE_METHODS(UInt32, uint32, uint32_t, UPB_UINT32ALT_T) +#endif + +#ifdef UPB_TWO_64BIT_TYPES +TYPE_METHODS(Int64, int64, int64_t, UPB_INT64ALT_T) +TYPE_METHODS(UInt64, uint64, uint64_t, UPB_UINT64ALT_T) +#endif +#undef TYPE_METHODS + +template <> struct CanonicalType { + typedef Status* Type; +}; + +template struct ReturnOf; + +template +struct ReturnOf { + typedef R Return; +}; + +template +struct ReturnOf { + typedef R Return; +}; + +template +struct ReturnOf { + typedef R Return; +}; + +template +struct ReturnOf { + typedef R Return; +}; + + +template +template +inline Handler::Handler(F func) + : registered_(false), + cleanup_data_(func.GetData()), + cleanup_func_(func.GetCleanup()) { + attr_.handler_data = func.GetData(); + typedef typename ReturnOf::Return Return; + typedef typename ConvertParams::Func ConvertedParamsFunc; + typedef typename MaybeWrapReturn::Func + ReturnWrappedFunc; + handler_ = ReturnWrappedFunc().Call; + + /* Set attributes based on what templates can statically tell us about the + * user's function. */ + + /* If the original function returns void, then we know that we wrapped it to + * always return ok. */ + bool always_ok = is_same::value; + attr_.alwaysok = always_ok; + + /* Closure parameter and return type. */ + attr_.closure_type = UniquePtrForType(); + + /* We use the closure type (from the first parameter) if the return type is + * void or bool, since these are the two cases we wrap to return the closure's + * type anyway. + * + * This is all nonsense for non START* handlers, but it doesn't matter because + * in that case the value will be ignored. */ + typedef typename FirstUnlessVoidOrBool::value + EffectiveReturn; + attr_.return_closure_type = UniquePtrForType(); +} + +template +inline void Handler::AddCleanup(upb_handlers* h) const { + UPB_ASSERT(!registered_); + registered_ = true; + if (cleanup_func_) { + bool ok = upb_handlers_addcleanup(h, cleanup_data_, cleanup_func_); + UPB_ASSERT(ok); + } +} + +} /* namespace upb */ + +#endif /* __cplusplus */ + + +#undef UPB_TWO_32BIT_TYPES +#undef UPB_TWO_64BIT_TYPES +#undef UPB_INT32_T +#undef UPB_UINT32_T +#undef UPB_INT32ALT_T +#undef UPB_UINT32ALT_T +#undef UPB_INT64_T +#undef UPB_UINT64_T +#undef UPB_INT64ALT_T +#undef UPB_UINT64ALT_T + +#include "upb/port_undef.inc" + +#endif /* UPB_HANDLERS_INL_H_ */ diff --git a/upb/handlers.c b/upb/handlers.c new file mode 100644 index 00000000000..844a3601765 --- /dev/null +++ b/upb/handlers.c @@ -0,0 +1,567 @@ +/* +** TODO(haberman): it's unclear whether a lot of the consistency checks should +** UPB_ASSERT() or return false. +*/ + +#include "upb/handlers.h" + +#include + +#include "upb/sink.h" + +#include "upb/port_def.inc" + +struct upb_handlers { + upb_handlercache *cache; + const upb_msgdef *msg; + const upb_handlers **sub; + const void *top_closure_type; + upb_handlers_tabent table[1]; /* Dynamically-sized field handler array. */ +}; + +static void *upb_calloc(upb_arena *arena, size_t size) { + void *mem = upb_malloc(upb_arena_alloc(arena), size); + if (mem) { + memset(mem, 0, size); + } + return mem; +} + +/* Defined for the sole purpose of having a unique pointer value for + * UPB_NO_CLOSURE. */ +char _upb_noclosure; + +/* Given a selector for a STARTSUBMSG handler, resolves to a pointer to the + * subhandlers for this submessage field. */ +#define SUBH(h, selector) (h->sub[selector]) + +/* The selector for a submessage field is the field index. */ +#define SUBH_F(h, f) SUBH(h, upb_fielddef_index(f)) + +static int32_t trygetsel(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + upb_selector_t sel; + bool ok; + + ok = upb_handlers_getselector(f, type, &sel); + + UPB_ASSERT(upb_handlers_msgdef(h) == upb_fielddef_containingtype(f)); + UPB_ASSERT(ok); + + return sel; +} + +static upb_selector_t handlers_getsel(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + int32_t sel = trygetsel(h, f, type); + UPB_ASSERT(sel >= 0); + return sel; +} + +static const void **returntype(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + return &h->table[handlers_getsel(h, f, type)].attr.return_closure_type; +} + +static bool doset(upb_handlers *h, int32_t sel, const upb_fielddef *f, + upb_handlertype_t type, upb_func *func, + const upb_handlerattr *attr) { + upb_handlerattr set_attr = UPB_HANDLERATTR_INIT; + const void *closure_type; + const void **context_closure_type; + + UPB_ASSERT(!h->table[sel].func); + + if (attr) { + set_attr = *attr; + } + + /* Check that the given closure type matches the closure type that has been + * established for this context (if any). */ + closure_type = set_attr.closure_type; + + if (type == UPB_HANDLER_STRING) { + context_closure_type = returntype(h, f, UPB_HANDLER_STARTSTR); + } else if (f && upb_fielddef_isseq(f) && + type != UPB_HANDLER_STARTSEQ && + type != UPB_HANDLER_ENDSEQ) { + context_closure_type = returntype(h, f, UPB_HANDLER_STARTSEQ); + } else { + context_closure_type = &h->top_closure_type; + } + + if (closure_type && *context_closure_type && + closure_type != *context_closure_type) { + return false; + } + + if (closure_type) + *context_closure_type = closure_type; + + /* If this is a STARTSEQ or STARTSTR handler, check that the returned pointer + * matches any pre-existing expectations about what type is expected. */ + if (type == UPB_HANDLER_STARTSEQ || type == UPB_HANDLER_STARTSTR) { + const void *return_type = set_attr.return_closure_type; + const void *table_return_type = h->table[sel].attr.return_closure_type; + if (return_type && table_return_type && return_type != table_return_type) { + return false; + } + + if (table_return_type && !return_type) { + set_attr.return_closure_type = table_return_type; + } + } + + h->table[sel].func = (upb_func*)func; + h->table[sel].attr = set_attr; + return true; +} + +/* Returns the effective closure type for this handler (which will propagate + * from outer frames if this frame has no START* handler). Not implemented for + * UPB_HANDLER_STRING at the moment since this is not needed. Returns NULL is + * the effective closure type is unspecified (either no handler was registered + * to specify it or the handler that was registered did not specify the closure + * type). */ +const void *effective_closure_type(upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + const void *ret; + upb_selector_t sel; + + UPB_ASSERT(type != UPB_HANDLER_STRING); + ret = h->top_closure_type; + + if (upb_fielddef_isseq(f) && + type != UPB_HANDLER_STARTSEQ && + type != UPB_HANDLER_ENDSEQ && + h->table[sel = handlers_getsel(h, f, UPB_HANDLER_STARTSEQ)].func) { + ret = h->table[sel].attr.return_closure_type; + } + + if (type == UPB_HANDLER_STRING && + h->table[sel = handlers_getsel(h, f, UPB_HANDLER_STARTSTR)].func) { + ret = h->table[sel].attr.return_closure_type; + } + + /* The effective type of the submessage; not used yet. + * if (type == SUBMESSAGE && + * h->table[sel = handlers_getsel(h, f, UPB_HANDLER_STARTSUBMSG)].func) { + * ret = h->table[sel].attr.return_closure_type; + * } */ + + return ret; +} + +/* Checks whether the START* handler specified by f & type is missing even + * though it is required to convert the established type of an outer frame + * ("closure_type") into the established type of an inner frame (represented in + * the return closure type of this handler's attr. */ +bool checkstart(upb_handlers *h, const upb_fielddef *f, upb_handlertype_t type, + upb_status *status) { + const void *closure_type; + const upb_handlerattr *attr; + const void *return_closure_type; + + upb_selector_t sel = handlers_getsel(h, f, type); + if (h->table[sel].func) return true; + closure_type = effective_closure_type(h, f, type); + attr = &h->table[sel].attr; + return_closure_type = attr->return_closure_type; + if (closure_type && return_closure_type && + closure_type != return_closure_type) { + return false; + } + return true; +} + +static upb_handlers *upb_handlers_new(const upb_msgdef *md, + upb_handlercache *cache, + upb_arena *arena) { + int extra; + upb_handlers *h; + + extra = sizeof(upb_handlers_tabent) * (upb_msgdef_selectorcount(md) - 1); + h = upb_calloc(arena, sizeof(*h) + extra); + if (!h) return NULL; + + h->cache = cache; + h->msg = md; + + if (upb_msgdef_submsgfieldcount(md) > 0) { + size_t bytes = upb_msgdef_submsgfieldcount(md) * sizeof(*h->sub); + h->sub = upb_calloc(arena, bytes); + if (!h->sub) return NULL; + } else { + h->sub = 0; + } + + /* calloc() above initialized all handlers to NULL. */ + return h; +} + +/* Public interface ***********************************************************/ + +#define SETTER(name, handlerctype, handlertype) \ + bool upb_handlers_set##name(upb_handlers *h, const upb_fielddef *f, \ + handlerctype func, \ + const upb_handlerattr *attr) { \ + int32_t sel = trygetsel(h, f, handlertype); \ + return doset(h, sel, f, handlertype, (upb_func *)func, attr); \ + } + +SETTER(int32, upb_int32_handlerfunc*, UPB_HANDLER_INT32) +SETTER(int64, upb_int64_handlerfunc*, UPB_HANDLER_INT64) +SETTER(uint32, upb_uint32_handlerfunc*, UPB_HANDLER_UINT32) +SETTER(uint64, upb_uint64_handlerfunc*, UPB_HANDLER_UINT64) +SETTER(float, upb_float_handlerfunc*, UPB_HANDLER_FLOAT) +SETTER(double, upb_double_handlerfunc*, UPB_HANDLER_DOUBLE) +SETTER(bool, upb_bool_handlerfunc*, UPB_HANDLER_BOOL) +SETTER(startstr, upb_startstr_handlerfunc*, UPB_HANDLER_STARTSTR) +SETTER(string, upb_string_handlerfunc*, UPB_HANDLER_STRING) +SETTER(endstr, upb_endfield_handlerfunc*, UPB_HANDLER_ENDSTR) +SETTER(startseq, upb_startfield_handlerfunc*, UPB_HANDLER_STARTSEQ) +SETTER(startsubmsg, upb_startfield_handlerfunc*, UPB_HANDLER_STARTSUBMSG) +SETTER(endsubmsg, upb_endfield_handlerfunc*, UPB_HANDLER_ENDSUBMSG) +SETTER(endseq, upb_endfield_handlerfunc*, UPB_HANDLER_ENDSEQ) + +#undef SETTER + +bool upb_handlers_setunknown(upb_handlers *h, upb_unknown_handlerfunc *func, + const upb_handlerattr *attr) { + return doset(h, UPB_UNKNOWN_SELECTOR, NULL, UPB_HANDLER_INT32, + (upb_func *)func, attr); +} + +bool upb_handlers_setstartmsg(upb_handlers *h, upb_startmsg_handlerfunc *func, + const upb_handlerattr *attr) { + return doset(h, UPB_STARTMSG_SELECTOR, NULL, UPB_HANDLER_INT32, + (upb_func *)func, attr); +} + +bool upb_handlers_setendmsg(upb_handlers *h, upb_endmsg_handlerfunc *func, + const upb_handlerattr *attr) { + return doset(h, UPB_ENDMSG_SELECTOR, NULL, UPB_HANDLER_INT32, + (upb_func *)func, attr); +} + +bool upb_handlers_setsubhandlers(upb_handlers *h, const upb_fielddef *f, + const upb_handlers *sub) { + UPB_ASSERT(sub); + UPB_ASSERT(upb_fielddef_issubmsg(f)); + if (SUBH_F(h, f)) return false; /* Can't reset. */ + if (upb_handlers_msgdef(sub) != upb_fielddef_msgsubdef(f)) { + return false; + } + SUBH_F(h, f) = sub; + return true; +} + +const upb_handlers *upb_handlers_getsubhandlers(const upb_handlers *h, + const upb_fielddef *f) { + UPB_ASSERT(upb_fielddef_issubmsg(f)); + return SUBH_F(h, f); +} + +upb_func *upb_handlers_gethandler(const upb_handlers *h, upb_selector_t s, + const void **handler_data) { + upb_func *ret = (upb_func *)h->table[s].func; + if (ret && handler_data) { + *handler_data = h->table[s].attr.handler_data; + } + return ret; +} + +bool upb_handlers_getattr(const upb_handlers *h, upb_selector_t sel, + upb_handlerattr *attr) { + if (!upb_handlers_gethandler(h, sel, NULL)) + return false; + *attr = h->table[sel].attr; + return true; +} + +const upb_handlers *upb_handlers_getsubhandlers_sel(const upb_handlers *h, + upb_selector_t sel) { + /* STARTSUBMSG selector in sel is the field's selector base. */ + return SUBH(h, sel - UPB_STATIC_SELECTOR_COUNT); +} + +const upb_msgdef *upb_handlers_msgdef(const upb_handlers *h) { return h->msg; } + +bool upb_handlers_addcleanup(upb_handlers *h, void *p, upb_handlerfree *func) { + return upb_handlercache_addcleanup(h->cache, p, func); +} + +upb_handlertype_t upb_handlers_getprimitivehandlertype(const upb_fielddef *f) { + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_ENUM: return UPB_HANDLER_INT32; + case UPB_TYPE_INT64: return UPB_HANDLER_INT64; + case UPB_TYPE_UINT32: return UPB_HANDLER_UINT32; + case UPB_TYPE_UINT64: return UPB_HANDLER_UINT64; + case UPB_TYPE_FLOAT: return UPB_HANDLER_FLOAT; + case UPB_TYPE_DOUBLE: return UPB_HANDLER_DOUBLE; + case UPB_TYPE_BOOL: return UPB_HANDLER_BOOL; + default: UPB_ASSERT(false); return -1; /* Invalid input. */ + } +} + +bool upb_handlers_getselector(const upb_fielddef *f, upb_handlertype_t type, + upb_selector_t *s) { + uint32_t selector_base = upb_fielddef_selectorbase(f); + switch (type) { + case UPB_HANDLER_INT32: + case UPB_HANDLER_INT64: + case UPB_HANDLER_UINT32: + case UPB_HANDLER_UINT64: + case UPB_HANDLER_FLOAT: + case UPB_HANDLER_DOUBLE: + case UPB_HANDLER_BOOL: + if (!upb_fielddef_isprimitive(f) || + upb_handlers_getprimitivehandlertype(f) != type) + return false; + *s = selector_base; + break; + case UPB_HANDLER_STRING: + if (upb_fielddef_isstring(f)) { + *s = selector_base; + } else if (upb_fielddef_lazy(f)) { + *s = selector_base + 3; + } else { + return false; + } + break; + case UPB_HANDLER_STARTSTR: + if (upb_fielddef_isstring(f) || upb_fielddef_lazy(f)) { + *s = selector_base + 1; + } else { + return false; + } + break; + case UPB_HANDLER_ENDSTR: + if (upb_fielddef_isstring(f) || upb_fielddef_lazy(f)) { + *s = selector_base + 2; + } else { + return false; + } + break; + case UPB_HANDLER_STARTSEQ: + if (!upb_fielddef_isseq(f)) return false; + *s = selector_base - 2; + break; + case UPB_HANDLER_ENDSEQ: + if (!upb_fielddef_isseq(f)) return false; + *s = selector_base - 1; + break; + case UPB_HANDLER_STARTSUBMSG: + if (!upb_fielddef_issubmsg(f)) return false; + /* Selectors for STARTSUBMSG are at the beginning of the table so that the + * selector can also be used as an index into the "sub" array of + * subhandlers. The indexes for the two into these two tables are the + * same, except that in the handler table the static selectors come first. */ + *s = upb_fielddef_index(f) + UPB_STATIC_SELECTOR_COUNT; + break; + case UPB_HANDLER_ENDSUBMSG: + if (!upb_fielddef_issubmsg(f)) return false; + *s = selector_base; + break; + } + UPB_ASSERT((size_t)*s < upb_msgdef_selectorcount(upb_fielddef_containingtype(f))); + return true; +} + +/* upb_handlercache ***********************************************************/ + +struct upb_handlercache { + upb_arena *arena; + upb_inttable tab; /* maps upb_msgdef* -> upb_handlers*. */ + upb_handlers_callback *callback; + const void *closure; +}; + +const upb_handlers *upb_handlercache_get(upb_handlercache *c, + const upb_msgdef *md) { + upb_msg_field_iter i; + upb_value v; + upb_handlers *h; + + if (upb_inttable_lookupptr(&c->tab, md, &v)) { + return upb_value_getptr(v); + } + + h = upb_handlers_new(md, c, c->arena); + v = upb_value_ptr(h); + + if (!h) return NULL; + if (!upb_inttable_insertptr(&c->tab, md, v)) return NULL; + + c->callback(c->closure, h); + + /* For each submessage field, get or create a handlers object and set it as + * the subhandlers. */ + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + + if (upb_fielddef_issubmsg(f)) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + const upb_handlers *sub_mh = upb_handlercache_get(c, subdef); + + if (!sub_mh) return NULL; + + upb_handlers_setsubhandlers(h, f, sub_mh); + } + } + + return h; +} + + +upb_handlercache *upb_handlercache_new(upb_handlers_callback *callback, + const void *closure) { + upb_handlercache *cache = upb_gmalloc(sizeof(*cache)); + + if (!cache) return NULL; + + cache->arena = upb_arena_new(); + + cache->callback = callback; + cache->closure = closure; + + if (!upb_inttable_init(&cache->tab, UPB_CTYPE_PTR)) goto oom; + + return cache; + +oom: + upb_gfree(cache); + return NULL; +} + +void upb_handlercache_free(upb_handlercache *cache) { + upb_inttable_uninit(&cache->tab); + upb_arena_free(cache->arena); + upb_gfree(cache); +} + +bool upb_handlercache_addcleanup(upb_handlercache *c, void *p, + upb_handlerfree *func) { + return upb_arena_addcleanup(c->arena, p, func); +} + +/* upb_byteshandler ***********************************************************/ + +bool upb_byteshandler_setstartstr(upb_byteshandler *h, + upb_startstr_handlerfunc *func, void *d) { + h->table[UPB_STARTSTR_SELECTOR].func = (upb_func*)func; + h->table[UPB_STARTSTR_SELECTOR].attr.handler_data = d; + return true; +} + +bool upb_byteshandler_setstring(upb_byteshandler *h, + upb_string_handlerfunc *func, void *d) { + h->table[UPB_STRING_SELECTOR].func = (upb_func*)func; + h->table[UPB_STRING_SELECTOR].attr.handler_data = d; + return true; +} + +bool upb_byteshandler_setendstr(upb_byteshandler *h, + upb_endfield_handlerfunc *func, void *d) { + h->table[UPB_ENDSTR_SELECTOR].func = (upb_func*)func; + h->table[UPB_ENDSTR_SELECTOR].attr.handler_data = d; + return true; +} + +/** Handlers for upb_msg ******************************************************/ + +typedef struct { + size_t offset; + int32_t hasbit; +} upb_msg_handlerdata; + +/* Fallback implementation if the handler is not specialized by the producer. */ +#define MSG_WRITER(type, ctype) \ + bool upb_msg_set ## type (void *c, const void *hd, ctype val) { \ + uint8_t *m = c; \ + const upb_msg_handlerdata *d = hd; \ + if (d->hasbit > 0) \ + *(uint8_t*)&m[d->hasbit / 8] |= 1 << (d->hasbit % 8); \ + *(ctype*)&m[d->offset] = val; \ + return true; \ + } \ + +MSG_WRITER(double, double) +MSG_WRITER(float, float) +MSG_WRITER(int32, int32_t) +MSG_WRITER(int64, int64_t) +MSG_WRITER(uint32, uint32_t) +MSG_WRITER(uint64, uint64_t) +MSG_WRITER(bool, bool) + +bool upb_msg_setscalarhandler(upb_handlers *h, const upb_fielddef *f, + size_t offset, int32_t hasbit) { + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + bool ok; + + upb_msg_handlerdata *d = upb_gmalloc(sizeof(*d)); + if (!d) return false; + d->offset = offset; + d->hasbit = hasbit; + + attr.handler_data = d; + attr.alwaysok = true; + upb_handlers_addcleanup(h, d, upb_gfree); + +#define TYPE(u, l) \ + case UPB_TYPE_##u: \ + ok = upb_handlers_set##l(h, f, upb_msg_set##l, &attr); break; + + ok = false; + + switch (upb_fielddef_type(f)) { + TYPE(INT64, int64); + TYPE(INT32, int32); + TYPE(ENUM, int32); + TYPE(UINT64, uint64); + TYPE(UINT32, uint32); + TYPE(DOUBLE, double); + TYPE(FLOAT, float); + TYPE(BOOL, bool); + default: UPB_ASSERT(false); break; + } +#undef TYPE + + return ok; +} + +bool upb_msg_getscalarhandlerdata(const upb_handlers *h, + upb_selector_t s, + upb_fieldtype_t *type, + size_t *offset, + int32_t *hasbit) { + const upb_msg_handlerdata *d; + const void *p; + upb_func *f = upb_handlers_gethandler(h, s, &p); + + if ((upb_int64_handlerfunc*)f == upb_msg_setint64) { + *type = UPB_TYPE_INT64; + } else if ((upb_int32_handlerfunc*)f == upb_msg_setint32) { + *type = UPB_TYPE_INT32; + } else if ((upb_uint64_handlerfunc*)f == upb_msg_setuint64) { + *type = UPB_TYPE_UINT64; + } else if ((upb_uint32_handlerfunc*)f == upb_msg_setuint32) { + *type = UPB_TYPE_UINT32; + } else if ((upb_double_handlerfunc*)f == upb_msg_setdouble) { + *type = UPB_TYPE_DOUBLE; + } else if ((upb_float_handlerfunc*)f == upb_msg_setfloat) { + *type = UPB_TYPE_FLOAT; + } else if ((upb_bool_handlerfunc*)f == upb_msg_setbool) { + *type = UPB_TYPE_BOOL; + } else { + return false; + } + + d = p; + *offset = d->offset; + *hasbit = d->hasbit; + return true; +} diff --git a/upb/handlers.h b/upb/handlers.h new file mode 100644 index 00000000000..2d2380b634b --- /dev/null +++ b/upb/handlers.h @@ -0,0 +1,732 @@ +/* +** upb::Handlers (upb_handlers) +** +** A upb_handlers is like a virtual table for a upb_msgdef. Each field of the +** message can have associated functions that will be called when we are +** parsing or visiting a stream of data. This is similar to how handlers work +** in SAX (the Simple API for XML). +** +** The handlers have no idea where the data is coming from, so a single set of +** handlers could be used with two completely different data sources (for +** example, a parser and a visitor over in-memory objects). This decoupling is +** the most important feature of upb, because it allows parsers and serializers +** to be highly reusable. +** +** This is a mixed C/C++ interface that offers a full API to both languages. +** See the top-level README for more information. +*/ + +#ifndef UPB_HANDLERS_H +#define UPB_HANDLERS_H + +#include "upb/def.h" +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +namespace upb { +class HandlersPtr; +class HandlerCache; +template class Handler; +template struct CanonicalType; +} /* namespace upb */ +#endif + + +/* The maximum depth that the handler graph can have. This is a resource limit + * for the C stack since we sometimes need to recursively traverse the graph. + * Cycles are ok; the traversal will stop when it detects a cycle, but we must + * hit the cycle before the maximum depth is reached. + * + * If having a single static limit is too inflexible, we can add another variant + * of Handlers::Freeze that allows specifying this as a parameter. */ +#define UPB_MAX_HANDLER_DEPTH 64 + +/* All the different types of handlers that can be registered. + * Only needed for the advanced functions in upb::Handlers. */ +typedef enum { + UPB_HANDLER_INT32, + UPB_HANDLER_INT64, + UPB_HANDLER_UINT32, + UPB_HANDLER_UINT64, + UPB_HANDLER_FLOAT, + UPB_HANDLER_DOUBLE, + UPB_HANDLER_BOOL, + UPB_HANDLER_STARTSTR, + UPB_HANDLER_STRING, + UPB_HANDLER_ENDSTR, + UPB_HANDLER_STARTSUBMSG, + UPB_HANDLER_ENDSUBMSG, + UPB_HANDLER_STARTSEQ, + UPB_HANDLER_ENDSEQ +} upb_handlertype_t; + +#define UPB_HANDLER_MAX (UPB_HANDLER_ENDSEQ+1) + +#define UPB_BREAK NULL + +/* A convenient definition for when no closure is needed. */ +extern char _upb_noclosure; +#define UPB_NO_CLOSURE &_upb_noclosure + +/* A selector refers to a specific field handler in the Handlers object + * (for example: the STARTSUBMSG handler for field "field15"). */ +typedef int32_t upb_selector_t; + +/* Static selectors for upb::Handlers. */ +#define UPB_STARTMSG_SELECTOR 0 +#define UPB_ENDMSG_SELECTOR 1 +#define UPB_UNKNOWN_SELECTOR 2 +#define UPB_STATIC_SELECTOR_COUNT 3 /* Warning: also in upb/def.c. */ + +/* Static selectors for upb::BytesHandler. */ +#define UPB_STARTSTR_SELECTOR 0 +#define UPB_STRING_SELECTOR 1 +#define UPB_ENDSTR_SELECTOR 2 + +#ifdef __cplusplus +template const void *UniquePtrForType() { + static const char ch = 0; + return &ch; +} +#endif + +/* upb_handlers ************************************************************/ + +/* Handler attributes, to be registered with the handler itself. */ +typedef struct { + const void *handler_data; + const void *closure_type; + const void *return_closure_type; + bool alwaysok; +} upb_handlerattr; + +#define UPB_HANDLERATTR_INIT {NULL, NULL, NULL, false} + +/* Bufhandle, data passed along with a buffer to indicate its provenance. */ +typedef struct { + /* The beginning of the buffer. This may be different than the pointer + * passed to a StringBuf handler because the handler may receive data + * that is from the middle or end of a larger buffer. */ + const char *buf; + + /* The offset within the attached object where this buffer begins. Only + * meaningful if there is an attached object. */ + size_t objofs; + + /* The attached object (if any) and a pointer representing its type. */ + const void *obj; + const void *objtype; + +#ifdef __cplusplus + template + void SetAttachedObject(const T* _obj) { + obj = _obj; + objtype = UniquePtrForType(); + } + + template + const T *GetAttachedObject() const { + return objtype == UniquePtrForType() ? static_cast(obj) + : NULL; + } +#endif +} upb_bufhandle; + +#define UPB_BUFHANDLE_INIT {NULL, 0, NULL, NULL} + +/* Handler function typedefs. */ +typedef void upb_handlerfree(void *d); +typedef bool upb_unknown_handlerfunc(void *c, const void *hd, const char *buf, + size_t n); +typedef bool upb_startmsg_handlerfunc(void *c, const void*); +typedef bool upb_endmsg_handlerfunc(void *c, const void *, upb_status *status); +typedef void* upb_startfield_handlerfunc(void *c, const void *hd); +typedef bool upb_endfield_handlerfunc(void *c, const void *hd); +typedef bool upb_int32_handlerfunc(void *c, const void *hd, int32_t val); +typedef bool upb_int64_handlerfunc(void *c, const void *hd, int64_t val); +typedef bool upb_uint32_handlerfunc(void *c, const void *hd, uint32_t val); +typedef bool upb_uint64_handlerfunc(void *c, const void *hd, uint64_t val); +typedef bool upb_float_handlerfunc(void *c, const void *hd, float val); +typedef bool upb_double_handlerfunc(void *c, const void *hd, double val); +typedef bool upb_bool_handlerfunc(void *c, const void *hd, bool val); +typedef void *upb_startstr_handlerfunc(void *c, const void *hd, + size_t size_hint); +typedef size_t upb_string_handlerfunc(void *c, const void *hd, const char *buf, + size_t n, const upb_bufhandle* handle); + +struct upb_handlers; +typedef struct upb_handlers upb_handlers; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Mutating accessors. */ +const upb_status *upb_handlers_status(upb_handlers *h); +void upb_handlers_clearerr(upb_handlers *h); +const upb_msgdef *upb_handlers_msgdef(const upb_handlers *h); +bool upb_handlers_addcleanup(upb_handlers *h, void *p, upb_handlerfree *hfree); +bool upb_handlers_setunknown(upb_handlers *h, upb_unknown_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartmsg(upb_handlers *h, upb_startmsg_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendmsg(upb_handlers *h, upb_endmsg_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setint32(upb_handlers *h, const upb_fielddef *f, + upb_int32_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setint64(upb_handlers *h, const upb_fielddef *f, + upb_int64_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setuint32(upb_handlers *h, const upb_fielddef *f, + upb_uint32_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setuint64(upb_handlers *h, const upb_fielddef *f, + upb_uint64_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setfloat(upb_handlers *h, const upb_fielddef *f, + upb_float_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setdouble(upb_handlers *h, const upb_fielddef *f, + upb_double_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setbool(upb_handlers *h, const upb_fielddef *f, + upb_bool_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartstr(upb_handlers *h, const upb_fielddef *f, + upb_startstr_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstring(upb_handlers *h, const upb_fielddef *f, + upb_string_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendstr(upb_handlers *h, const upb_fielddef *f, + upb_endfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartseq(upb_handlers *h, const upb_fielddef *f, + upb_startfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setstartsubmsg(upb_handlers *h, const upb_fielddef *f, + upb_startfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendsubmsg(upb_handlers *h, const upb_fielddef *f, + upb_endfield_handlerfunc *func, + const upb_handlerattr *attr); +bool upb_handlers_setendseq(upb_handlers *h, const upb_fielddef *f, + upb_endfield_handlerfunc *func, + const upb_handlerattr *attr); + +/* Read-only accessors. */ +const upb_handlers *upb_handlers_getsubhandlers(const upb_handlers *h, + const upb_fielddef *f); +const upb_handlers *upb_handlers_getsubhandlers_sel(const upb_handlers *h, + upb_selector_t sel); +upb_func *upb_handlers_gethandler(const upb_handlers *h, upb_selector_t s, + const void **handler_data); +bool upb_handlers_getattr(const upb_handlers *h, upb_selector_t s, + upb_handlerattr *attr); + +/* "Static" methods */ +upb_handlertype_t upb_handlers_getprimitivehandlertype(const upb_fielddef *f); +bool upb_handlers_getselector(const upb_fielddef *f, upb_handlertype_t type, + upb_selector_t *s); +UPB_INLINE upb_selector_t upb_handlers_getendselector(upb_selector_t start) { + return start + 1; +} + +#ifdef __cplusplus +} /* extern "C" */ + +namespace upb { +typedef upb_handlers Handlers; +} + +/* Convenience macros for creating a Handler object that is wrapped with a + * type-safe wrapper function that converts the "void*" parameters/returns + * of the underlying C API into nice C++ function. + * + * Sample usage: + * void OnValue1(MyClosure* c, const MyHandlerData* d, int32_t val) { + * // do stuff ... + * } + * + * // Handler that doesn't need any data bound to it. + * void OnValue2(MyClosure* c, int32_t val) { + * // do stuff ... + * } + * + * // Handler that returns bool so it can return failure if necessary. + * bool OnValue3(MyClosure* c, int32_t val) { + * // do stuff ... + * return ok; + * } + * + * // Member function handler. + * class MyClosure { + * public: + * void OnValue(int32_t val) { + * // do stuff ... + * } + * }; + * + * // Takes ownership of the MyHandlerData. + * handlers->SetInt32Handler(f1, UpbBind(OnValue1, new MyHandlerData(...))); + * handlers->SetInt32Handler(f2, UpbMakeHandler(OnValue2)); + * handlers->SetInt32Handler(f1, UpbMakeHandler(OnValue3)); + * handlers->SetInt32Handler(f2, UpbMakeHandler(&MyClosure::OnValue)); + */ + +/* In C++11, the "template" disambiguator can appear even outside templates, + * so all calls can safely use this pair of macros. */ + +#define UpbMakeHandler(f) upb::MatchFunc(f).template GetFunc() + +/* We have to be careful to only evaluate "d" once. */ +#define UpbBind(f, d) upb::MatchFunc(f).template GetFunc((d)) + +/* Handler: a struct that contains the (handler, data, deleter) tuple that is + * used to register all handlers. Users can Make() these directly but it's + * more convenient to use the UpbMakeHandler/UpbBind macros above. */ +template class upb::Handler { + public: + /* The underlying, handler function signature that upb uses internally. */ + typedef T FuncPtr; + + /* Intentionally implicit. */ + template Handler(F func); + ~Handler() { UPB_ASSERT(registered_); } + + void AddCleanup(upb_handlers* h) const; + FuncPtr handler() const { return handler_; } + const upb_handlerattr& attr() const { return attr_; } + + private: + Handler(const Handler&) = delete; + Handler& operator=(const Handler&) = delete; + + FuncPtr handler_; + mutable upb_handlerattr attr_; + mutable bool registered_; + void *cleanup_data_; + upb_handlerfree *cleanup_func_; +}; + +/* A upb::Handlers object represents the set of handlers associated with a + * message in the graph of messages. You can think of it as a big virtual + * table with functions corresponding to all the events that can fire while + * parsing or visiting a message of a specific type. + * + * Any handlers that are not set behave as if they had successfully consumed + * the value. Any unset Start* handlers will propagate their closure to the + * inner frame. + * + * The easiest way to create the *Handler objects needed by the Set* methods is + * with the UpbBind() and UpbMakeHandler() macros; see below. */ +class upb::HandlersPtr { + public: + HandlersPtr(upb_handlers* ptr) : ptr_(ptr) {} + + upb_handlers* ptr() const { return ptr_; } + + typedef upb_selector_t Selector; + typedef upb_handlertype_t Type; + + typedef Handler StartFieldHandler; + typedef Handler EndFieldHandler; + typedef Handler StartMessageHandler; + typedef Handler + EndMessageHandler; + typedef Handler StartStringHandler; + typedef Handler + StringHandler; + + template struct ValueHandler { + typedef Handler H; + }; + + typedef ValueHandler::H Int32Handler; + typedef ValueHandler::H Int64Handler; + typedef ValueHandler::H UInt32Handler; + typedef ValueHandler::H UInt64Handler; + typedef ValueHandler::H FloatHandler; + typedef ValueHandler::H DoubleHandler; + typedef ValueHandler::H BoolHandler; + + /* Any function pointer can be converted to this and converted back to its + * correct type. */ + typedef void GenericFunction(); + + typedef void HandlersCallback(const void *closure, upb_handlers *h); + + /* Returns the msgdef associated with this handlers object. */ + MessageDefPtr message_def() const { + return MessageDefPtr(upb_handlers_msgdef(ptr())); + } + + /* Adds the given pointer and function to the list of cleanup functions that + * will be run when these handlers are freed. If this pointer has previously + * been registered, the function returns false and does nothing. */ + bool AddCleanup(void *ptr, upb_handlerfree *cleanup) { + return upb_handlers_addcleanup(ptr_, ptr, cleanup); + } + + /* Sets the startmsg handler for the message, which is defined as follows: + * + * bool startmsg(MyType* closure) { + * // Called when the message begins. Returns true if processing should + * // continue. + * return true; + * } + */ + bool SetStartMessageHandler(const StartMessageHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartmsg(ptr(), h.handler(), &h.attr()); + } + + /* Sets the endmsg handler for the message, which is defined as follows: + * + * bool endmsg(MyType* closure, upb_status *status) { + * // Called when processing of this message ends, whether in success or + * // failure. "status" indicates the final status of processing, and + * // can also be modified in-place to update the final status. + * } + */ + bool SetEndMessageHandler(const EndMessageHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setendmsg(ptr(), h.handler(), &h.attr()); + } + + /* Sets the value handler for the given field, which is defined as follows + * (this is for an int32 field; other field types will pass their native + * C/C++ type for "val"): + * + * bool OnValue(MyClosure* c, const MyHandlerData* d, int32_t val) { + * // Called when the field's value is encountered. "d" contains + * // whatever data was bound to this field when it was registered. + * // Returns true if processing should continue. + * return true; + * } + * + * handers->SetInt32Handler(f, UpbBind(OnValue, new MyHandlerData(...))); + * + * The value type must exactly match f->type(). + * For example, a handler that takes an int32_t parameter may only be used for + * fields of type UPB_TYPE_INT32 and UPB_TYPE_ENUM. + * + * Returns false if the handler failed to register; in this case the cleanup + * handler (if any) will be called immediately. + */ + bool SetInt32Handler(FieldDefPtr f, const Int32Handler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setint32(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetInt64Handler (FieldDefPtr f, const Int64Handler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setint64(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetUInt32Handler(FieldDefPtr f, const UInt32Handler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setuint32(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetUInt64Handler(FieldDefPtr f, const UInt64Handler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setuint64(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetFloatHandler (FieldDefPtr f, const FloatHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setfloat(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetDoubleHandler(FieldDefPtr f, const DoubleHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setdouble(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetBoolHandler(FieldDefPtr f, const BoolHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setbool(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Like the previous, but templated on the type on the value (ie. int32). + * This is mostly useful to call from other templates. To call this you must + * specify the template parameter explicitly, ie: + * h->SetValueHandler(f, UpbBind(MyHandler, MyData)); */ + template + bool SetValueHandler( + FieldDefPtr f, + const typename ValueHandler::Type>::H &handler); + + /* Sets handlers for a string field, which are defined as follows: + * + * MySubClosure* startstr(MyClosure* c, const MyHandlerData* d, + * size_t size_hint) { + * // Called when a string value begins. The return value indicates the + * // closure for the string. "size_hint" indicates the size of the + * // string if it is known, however if the string is length-delimited + * // and the end-of-string is not available size_hint will be zero. + * // This case is indistinguishable from the case where the size is + * // known to be zero. + * // + * // TODO(haberman): is it important to distinguish these cases? + * // If we had ssize_t as a type we could make -1 "unknown", but + * // ssize_t is POSIX (not ANSI) and therefore less portable. + * // In practice I suspect it won't be important to distinguish. + * return closure; + * } + * + * size_t str(MyClosure* closure, const MyHandlerData* d, + * const char *str, size_t len) { + * // Called for each buffer of string data; the multiple physical buffers + * // are all part of the same logical string. The return value indicates + * // how many bytes were consumed. If this number is less than "len", + * // this will also indicate that processing should be halted for now, + * // like returning false or UPB_BREAK from any other callback. If + * // number is greater than "len", the excess bytes will be skipped over + * // and not passed to the callback. + * return len; + * } + * + * bool endstr(MyClosure* c, const MyHandlerData* d) { + * // Called when a string value ends. Return value indicates whether + * // processing should continue. + * return true; + * } + */ + bool SetStartStringHandler(FieldDefPtr f, const StartStringHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartstr(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetStringHandler(FieldDefPtr f, const StringHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setstring(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + bool SetEndStringHandler(FieldDefPtr f, const EndFieldHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setendstr(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Sets the startseq handler, which is defined as follows: + * + * MySubClosure *startseq(MyClosure* c, const MyHandlerData* d) { + * // Called when a sequence (repeated field) begins. The returned + * // pointer indicates the closure for the sequence (or UPB_BREAK + * // to interrupt processing). + * return closure; + * } + * + * h->SetStartSequenceHandler(f, UpbBind(startseq, new MyHandlerData(...))); + * + * Returns "false" if "f" does not belong to this message or is not a + * repeated field. + */ + bool SetStartSequenceHandler(FieldDefPtr f, const StartFieldHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartseq(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Sets the startsubmsg handler for the given field, which is defined as + * follows: + * + * MySubClosure* startsubmsg(MyClosure* c, const MyHandlerData* d) { + * // Called when a submessage begins. The returned pointer indicates the + * // closure for the sequence (or UPB_BREAK to interrupt processing). + * return closure; + * } + * + * h->SetStartSubMessageHandler(f, UpbBind(startsubmsg, + * new MyHandlerData(...))); + * + * Returns "false" if "f" does not belong to this message or is not a + * submessage/group field. + */ + bool SetStartSubMessageHandler(FieldDefPtr f, const StartFieldHandler& h) { + h.AddCleanup(ptr()); + return upb_handlers_setstartsubmsg(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Sets the endsubmsg handler for the given field, which is defined as + * follows: + * + * bool endsubmsg(MyClosure* c, const MyHandlerData* d) { + * // Called when a submessage ends. Returns true to continue processing. + * return true; + * } + * + * Returns "false" if "f" does not belong to this message or is not a + * submessage/group field. + */ + bool SetEndSubMessageHandler(FieldDefPtr f, const EndFieldHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setendsubmsg(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + /* Starts the endsubseq handler for the given field, which is defined as + * follows: + * + * bool endseq(MyClosure* c, const MyHandlerData* d) { + * // Called when a sequence ends. Returns true continue processing. + * return true; + * } + * + * Returns "false" if "f" does not belong to this message or is not a + * repeated field. + */ + bool SetEndSequenceHandler(FieldDefPtr f, const EndFieldHandler &h) { + h.AddCleanup(ptr()); + return upb_handlers_setendseq(ptr(), f.ptr(), h.handler(), &h.attr()); + } + + private: + upb_handlers* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_handlercache ***********************************************************/ + +/* A upb_handlercache lazily builds and caches upb_handlers. You pass it a + * function (with optional closure) that can build handlers for a given + * message on-demand, and the cache maintains a map of msgdef->handlers. */ + +#ifdef __cplusplus +extern "C" { +#endif + +struct upb_handlercache; +typedef struct upb_handlercache upb_handlercache; + +typedef void upb_handlers_callback(const void *closure, upb_handlers *h); + +upb_handlercache *upb_handlercache_new(upb_handlers_callback *callback, + const void *closure); +void upb_handlercache_free(upb_handlercache *cache); +const upb_handlers *upb_handlercache_get(upb_handlercache *cache, + const upb_msgdef *md); +bool upb_handlercache_addcleanup(upb_handlercache *h, void *p, + upb_handlerfree *hfree); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::HandlerCache { + public: + HandlerCache(upb_handlers_callback *callback, const void *closure) + : ptr_(upb_handlercache_new(callback, closure), upb_handlercache_free) {} + HandlerCache(HandlerCache&&) = default; + HandlerCache& operator=(HandlerCache&&) = default; + HandlerCache(upb_handlercache* c) : ptr_(c, upb_handlercache_free) {} + + upb_handlercache* ptr() { return ptr_.get(); } + + const upb_handlers *Get(MessageDefPtr md) { + return upb_handlercache_get(ptr_.get(), md.ptr()); + } + + private: + std::unique_ptr ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_byteshandler ***********************************************************/ + +typedef struct { + upb_func *func; + + /* It is wasteful to include the entire attributes here: + * + * * Some of the information is redundant (like storing the closure type + * separately for each handler that must match). + * * Some of the info is only needed prior to freeze() (like closure types). + * * alignment padding wastes a lot of space for alwaysok_. + * + * If/when the size and locality of handlers is an issue, we can optimize this + * not to store the entire attr like this. We do not expose the table's + * layout to allow this optimization in the future. */ + upb_handlerattr attr; +} upb_handlers_tabent; + +#define UPB_TABENT_INIT {NULL, UPB_HANDLERATTR_INIT} + +typedef struct { + upb_handlers_tabent table[3]; +} upb_byteshandler; + +#define UPB_BYTESHANDLER_INIT \ + { \ + { UPB_TABENT_INIT, UPB_TABENT_INIT, UPB_TABENT_INIT } \ + } + +UPB_INLINE void upb_byteshandler_init(upb_byteshandler *handler) { + upb_byteshandler init = UPB_BYTESHANDLER_INIT; + *handler = init; +} + +#ifdef __cplusplus +extern "C" { +#endif + +/* Caller must ensure that "d" outlives the handlers. */ +bool upb_byteshandler_setstartstr(upb_byteshandler *h, + upb_startstr_handlerfunc *func, void *d); +bool upb_byteshandler_setstring(upb_byteshandler *h, + upb_string_handlerfunc *func, void *d); +bool upb_byteshandler_setendstr(upb_byteshandler *h, + upb_endfield_handlerfunc *func, void *d); + +#ifdef __cplusplus +} /* extern "C" */ + +namespace upb { +typedef upb_byteshandler BytesHandler; +} +#endif + +/** Message handlers ******************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* These are the handlers used internally by upb_msgfactory_getmergehandlers(). + * They write scalar data to a known offset from the message pointer. + * + * These would be trivial for anyone to implement themselves, but it's better + * to use these because some JITs will recognize and specialize these instead + * of actually calling the function. */ + +/* Sets a handler for the given primitive field that will write the data at the + * given offset. If hasbit > 0, also sets a hasbit at the given bit offset + * (addressing each byte low to high). */ +bool upb_msg_setscalarhandler(upb_handlers *h, + const upb_fielddef *f, + size_t offset, + int32_t hasbit); + +/* If the given handler is a msghandlers_primitive field, returns true and sets + * *type, *offset and *hasbit. Otherwise returns false. */ +bool upb_msg_getscalarhandlerdata(const upb_handlers *h, + upb_selector_t s, + upb_fieldtype_t *type, + size_t *offset, + int32_t *hasbit); + + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#include "upb/handlers-inl.h" + +#endif /* UPB_HANDLERS_H */ diff --git a/upb/json/parser.h b/upb/json/parser.h new file mode 100644 index 00000000000..d323c52ef8c --- /dev/null +++ b/upb/json/parser.h @@ -0,0 +1,140 @@ +/* +** upb::json::Parser (upb_json_parser) +** +** Parses JSON according to a specific schema. +** Support for parsing arbitrary JSON (schema-less) will be added later. +*/ + +#ifndef UPB_JSON_PARSER_H_ +#define UPB_JSON_PARSER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace json { +class CodeCache; +class ParserPtr; +class ParserMethodPtr; +} /* namespace json */ +} /* namespace upb */ +#endif + +/* upb_json_parsermethod ******************************************************/ + +struct upb_json_parsermethod; +typedef struct upb_json_parsermethod upb_json_parsermethod; + +#ifdef __cplusplus +extern "C" { +#endif + +const upb_byteshandler* upb_json_parsermethod_inputhandler( + const upb_json_parsermethod* m); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::json::ParserMethodPtr { + public: + ParserMethodPtr() : ptr_(nullptr) {} + ParserMethodPtr(const upb_json_parsermethod* ptr) : ptr_(ptr) {} + + const upb_json_parsermethod* ptr() const { return ptr_; } + + const BytesHandler* input_handler() const { + return upb_json_parsermethod_inputhandler(ptr()); + } + + private: + const upb_json_parsermethod* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_json_parser ************************************************************/ + +/* Preallocation hint: parser won't allocate more bytes than this when first + * constructed. This hint may be an overestimate for some build configurations. + * But if the parser library is upgraded without recompiling the application, + * it may be an underestimate. */ +#define UPB_JSON_PARSER_SIZE 5712 + +struct upb_json_parser; +typedef struct upb_json_parser upb_json_parser; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_json_parser* upb_json_parser_create(upb_arena* a, + const upb_json_parsermethod* m, + const upb_symtab* symtab, + upb_sink output, + upb_status *status, + bool ignore_json_unknown); +upb_bytessink upb_json_parser_input(upb_json_parser* p); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Parses an incoming BytesStream, pushing the results to the destination + * sink. */ +class upb::json::ParserPtr { + public: + ParserPtr(upb_json_parser* ptr) : ptr_(ptr) {} + + static ParserPtr Create(Arena* arena, ParserMethodPtr method, + SymbolTable* symtab, Sink output, Status* status, + bool ignore_json_unknown) { + upb_symtab* symtab_ptr = symtab ? symtab->ptr() : nullptr; + return ParserPtr(upb_json_parser_create( + arena->ptr(), method.ptr(), symtab_ptr, output.sink(), status->ptr(), + ignore_json_unknown)); + } + + BytesSink input() { return upb_json_parser_input(ptr_); } + + private: + upb_json_parser* ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_json_codecache *********************************************************/ + +/* Lazily builds and caches decoder methods that will push data to the given + * handlers. The upb_symtab object(s) must outlive this object. */ + +struct upb_json_codecache; +typedef struct upb_json_codecache upb_json_codecache; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_json_codecache *upb_json_codecache_new(void); +void upb_json_codecache_free(upb_json_codecache *cache); +const upb_json_parsermethod* upb_json_codecache_get(upb_json_codecache* cache, + const upb_msgdef* md); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::json::CodeCache { + public: + CodeCache() : ptr_(upb_json_codecache_new(), upb_json_codecache_free) {} + + /* Returns a DecoderMethod that can push data to the given handlers. + * If a suitable method already exists, it will be returned from the cache. */ + ParserMethodPtr Get(MessageDefPtr md) { + return upb_json_codecache_get(ptr_.get(), md.ptr()); + } + + private: + std::unique_ptr ptr_; +}; + +#endif + +#endif /* UPB_JSON_PARSER_H_ */ diff --git a/upb/json/parser.rl b/upb/json/parser.rl new file mode 100644 index 00000000000..2641dda31c3 --- /dev/null +++ b/upb/json/parser.rl @@ -0,0 +1,3017 @@ +/* +** upb::json::Parser (upb_json_parser) +** +** A parser that uses the Ragel State Machine Compiler to generate +** the finite automata. +** +** Ragel only natively handles regular languages, but we can manually +** program it a bit to handle context-free languages like JSON, by using +** the "fcall" and "fret" constructs. +** +** This parser can handle the basics, but needs several things to be fleshed +** out: +** +** - handling of unicode escape sequences (including high surrogate pairs). +** - properly check and report errors for unknown fields, stack overflow, +** improper array nesting (or lack of nesting). +** - handling of base64 sequences with padding characters. +** - handling of push-back (non-success returns from sink functions). +** - handling of keys/escape-sequences/etc that span input buffers. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "upb/json/parser.h" +#include "upb/pb/encoder.h" + +#include "upb/port_def.inc" + +#define UPB_JSON_MAX_DEPTH 64 + +/* Type of value message */ +enum { + VALUE_NULLVALUE = 0, + VALUE_NUMBERVALUE = 1, + VALUE_STRINGVALUE = 2, + VALUE_BOOLVALUE = 3, + VALUE_STRUCTVALUE = 4, + VALUE_LISTVALUE = 5 +}; + +/* Forward declare */ +static bool is_top_level(upb_json_parser *p); +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type); +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type); + +static bool is_number_wrapper_object(upb_json_parser *p); +static bool does_number_wrapper_start(upb_json_parser *p); +static bool does_number_wrapper_end(upb_json_parser *p); + +static bool is_string_wrapper_object(upb_json_parser *p); +static bool does_string_wrapper_start(upb_json_parser *p); +static bool does_string_wrapper_end(upb_json_parser *p); + +static bool does_fieldmask_start(upb_json_parser *p); +static bool does_fieldmask_end(upb_json_parser *p); +static void start_fieldmask_object(upb_json_parser *p); +static void end_fieldmask_object(upb_json_parser *p); + +static void start_wrapper_object(upb_json_parser *p); +static void end_wrapper_object(upb_json_parser *p); + +static void start_value_object(upb_json_parser *p, int value_type); +static void end_value_object(upb_json_parser *p); + +static void start_listvalue_object(upb_json_parser *p); +static void end_listvalue_object(upb_json_parser *p); + +static void start_structvalue_object(upb_json_parser *p); +static void end_structvalue_object(upb_json_parser *p); + +static void start_object(upb_json_parser *p); +static void end_object(upb_json_parser *p); + +static void start_any_object(upb_json_parser *p, const char *ptr); +static bool end_any_object(upb_json_parser *p, const char *ptr); + +static bool start_subobject(upb_json_parser *p); +static void end_subobject(upb_json_parser *p); + +static void start_member(upb_json_parser *p); +static void end_member(upb_json_parser *p); +static bool end_membername(upb_json_parser *p); + +static void start_any_member(upb_json_parser *p, const char *ptr); +static void end_any_member(upb_json_parser *p, const char *ptr); +static bool end_any_membername(upb_json_parser *p); + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle); +static bool end(void *closure, const void *hd); + +static const char eof_ch = 'e'; + +/* stringsink */ +typedef struct { + upb_byteshandler handler; + upb_bytessink sink; + char *ptr; + size_t len, size; +} upb_stringsink; + + +static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) { + upb_stringsink *sink = _sink; + sink->len = 0; + UPB_UNUSED(hd); + UPB_UNUSED(size_hint); + return sink; +} + +static size_t stringsink_string(void *_sink, const void *hd, const char *ptr, + size_t len, const upb_bufhandle *handle) { + upb_stringsink *sink = _sink; + size_t new_size = sink->size; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + while (sink->len + len > new_size) { + new_size *= 2; + } + + if (new_size != sink->size) { + sink->ptr = realloc(sink->ptr, new_size); + sink->size = new_size; + } + + memcpy(sink->ptr + sink->len, ptr, len); + sink->len += len; + + return len; +} + +void upb_stringsink_init(upb_stringsink *sink) { + upb_byteshandler_init(&sink->handler); + upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL); + upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL); + + upb_bytessink_reset(&sink->sink, &sink->handler, sink); + + sink->size = 32; + sink->ptr = malloc(sink->size); + sink->len = 0; +} + +void upb_stringsink_uninit(upb_stringsink *sink) { free(sink->ptr); } + +typedef struct { + /* For encoding Any value field in binary format. */ + upb_handlercache *encoder_handlercache; + upb_stringsink stringsink; + + /* For decoding Any value field in json format. */ + upb_json_codecache *parser_codecache; + upb_sink sink; + upb_json_parser *parser; + + /* Mark the range of uninterpreted values in json input before type url. */ + const char *before_type_url_start; + const char *before_type_url_end; + + /* Mark the range of uninterpreted values in json input after type url. */ + const char *after_type_url_start; +} upb_jsonparser_any_frame; + +typedef struct { + upb_sink sink; + + /* The current message in which we're parsing, and the field whose value we're + * expecting next. */ + const upb_msgdef *m; + const upb_fielddef *f; + + /* The table mapping json name to fielddef for this message. */ + const upb_strtable *name_table; + + /* We are in a repeated-field context. We need this flag to decide whether to + * handle the array as a normal repeated field or a + * google.protobuf.ListValue/google.protobuf.Value. */ + bool is_repeated; + + /* We are in a repeated-field context, ready to emit mapentries as + * submessages. This flag alters the start-of-object (open-brace) behavior to + * begin a sequence of mapentry messages rather than a single submessage. */ + bool is_map; + + /* We are in a map-entry message context. This flag is set when parsing the + * value field of a single map entry and indicates to all value-field parsers + * (subobjects, strings, numbers, and bools) that the map-entry submessage + * should end as soon as the value is parsed. */ + bool is_mapentry; + + /* If |is_map| or |is_mapentry| is true, |mapfield| refers to the parent + * message's map field that we're currently parsing. This differs from |f| + * because |f| is the field in the *current* message (i.e., the map-entry + * message itself), not the parent's field that leads to this map. */ + const upb_fielddef *mapfield; + + /* We are in an Any message context. This flag is set when parsing the Any + * message and indicates to all field parsers (subobjects, strings, numbers, + * and bools) that the parsed field should be serialized as binary data or + * cached (type url not found yet). */ + bool is_any; + + /* The type of packed message in Any. */ + upb_jsonparser_any_frame *any_frame; + + /* True if the field to be parsed is unknown. */ + bool is_unknown_field; +} upb_jsonparser_frame; + +static void init_frame(upb_jsonparser_frame* frame) { + frame->m = NULL; + frame->f = NULL; + frame->name_table = NULL; + frame->is_repeated = false; + frame->is_map = false; + frame->is_mapentry = false; + frame->mapfield = NULL; + frame->is_any = false; + frame->any_frame = NULL; + frame->is_unknown_field = false; +} + +struct upb_json_parser { + upb_arena *arena; + const upb_json_parsermethod *method; + upb_bytessink input_; + + /* Stack to track the JSON scopes we are in. */ + upb_jsonparser_frame stack[UPB_JSON_MAX_DEPTH]; + upb_jsonparser_frame *top; + upb_jsonparser_frame *limit; + + upb_status *status; + + /* Ragel's internal parsing stack for the parsing state machine. */ + int current_state; + int parser_stack[UPB_JSON_MAX_DEPTH]; + int parser_top; + + /* The handle for the current buffer. */ + const upb_bufhandle *handle; + + /* Accumulate buffer. See details in parser.rl. */ + const char *accumulated; + size_t accumulated_len; + char *accumulate_buf; + size_t accumulate_buf_size; + + /* Multi-part text data. See details in parser.rl. */ + int multipart_state; + upb_selector_t string_selector; + + /* Input capture. See details in parser.rl. */ + const char *capture; + + /* Intermediate result of parsing a unicode escape sequence. */ + uint32_t digit; + + /* For resolve type url in Any. */ + const upb_symtab *symtab; + + /* Whether to proceed if unknown field is met. */ + bool ignore_json_unknown; + + /* Cache for parsing timestamp due to base and zone are handled in different + * handlers. */ + struct tm tm; +}; + +static upb_jsonparser_frame* start_jsonparser_frame(upb_json_parser *p) { + upb_jsonparser_frame *inner; + inner = p->top + 1; + init_frame(inner); + return inner; +} + +struct upb_json_codecache { + upb_arena *arena; + upb_inttable methods; /* upb_msgdef* -> upb_json_parsermethod* */ +}; + +struct upb_json_parsermethod { + const upb_json_codecache *cache; + upb_byteshandler input_handler_; + + /* Maps json_name -> fielddef */ + upb_strtable name_table; +}; + +#define PARSER_CHECK_RETURN(x) if (!(x)) return false + +static upb_jsonparser_any_frame *json_parser_any_frame_new( + upb_json_parser *p) { + upb_jsonparser_any_frame *frame; + + frame = upb_arena_malloc(p->arena, sizeof(upb_jsonparser_any_frame)); + + frame->encoder_handlercache = upb_pb_encoder_newcache(); + frame->parser_codecache = upb_json_codecache_new(); + frame->parser = NULL; + frame->before_type_url_start = NULL; + frame->before_type_url_end = NULL; + frame->after_type_url_start = NULL; + + upb_stringsink_init(&frame->stringsink); + + return frame; +} + +static void json_parser_any_frame_set_payload_type( + upb_json_parser *p, + upb_jsonparser_any_frame *frame, + const upb_msgdef *payload_type) { + const upb_handlers *h; + const upb_json_parsermethod *parser_method; + upb_pb_encoder *encoder; + + /* Initialize encoder. */ + h = upb_handlercache_get(frame->encoder_handlercache, payload_type); + encoder = upb_pb_encoder_create(p->arena, h, frame->stringsink.sink); + + /* Initialize parser. */ + parser_method = upb_json_codecache_get(frame->parser_codecache, payload_type); + upb_sink_reset(&frame->sink, h, encoder); + frame->parser = + upb_json_parser_create(p->arena, parser_method, p->symtab, frame->sink, + p->status, p->ignore_json_unknown); +} + +static void json_parser_any_frame_free(upb_jsonparser_any_frame *frame) { + upb_handlercache_free(frame->encoder_handlercache); + upb_json_codecache_free(frame->parser_codecache); + upb_stringsink_uninit(&frame->stringsink); +} + +static bool json_parser_any_frame_has_type_url( + upb_jsonparser_any_frame *frame) { + return frame->parser != NULL; +} + +static bool json_parser_any_frame_has_value_before_type_url( + upb_jsonparser_any_frame *frame) { + return frame->before_type_url_start != frame->before_type_url_end; +} + +static bool json_parser_any_frame_has_value_after_type_url( + upb_jsonparser_any_frame *frame) { + return frame->after_type_url_start != NULL; +} + +static bool json_parser_any_frame_has_value( + upb_jsonparser_any_frame *frame) { + return json_parser_any_frame_has_value_before_type_url(frame) || + json_parser_any_frame_has_value_after_type_url(frame); +} + +static void json_parser_any_frame_set_before_type_url_end( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (frame->parser == NULL) { + frame->before_type_url_end = ptr; + } +} + +static void json_parser_any_frame_set_after_type_url_start_once( + upb_jsonparser_any_frame *frame, + const char *ptr) { + if (json_parser_any_frame_has_type_url(frame) && + frame->after_type_url_start == NULL) { + frame->after_type_url_start = ptr; + } +} + +/* Used to signal that a capture has been suspended. */ +static char suspend_capture; + +static upb_selector_t getsel_for_handlertype(upb_json_parser *p, + upb_handlertype_t type) { + upb_selector_t sel; + bool ok = upb_handlers_getselector(p->top->f, type, &sel); + UPB_ASSERT(ok); + return sel; +} + +static upb_selector_t parser_getsel(upb_json_parser *p) { + return getsel_for_handlertype( + p, upb_handlers_getprimitivehandlertype(p->top->f)); +} + +static bool check_stack(upb_json_parser *p) { + if ((p->top + 1) == p->limit) { + upb_status_seterrmsg(p->status, "Nesting too deep"); + return false; + } + + return true; +} + +static void set_name_table(upb_json_parser *p, upb_jsonparser_frame *frame) { + upb_value v; + const upb_json_codecache *cache = p->method->cache; + bool ok; + const upb_json_parsermethod *method; + + ok = upb_inttable_lookupptr(&cache->methods, frame->m, &v); + UPB_ASSERT(ok); + method = upb_value_getconstptr(v); + + frame->name_table = &method->name_table; +} + +/* There are GCC/Clang built-ins for overflow checking which we could start + * using if there was any performance benefit to it. */ + +static bool checked_add(size_t a, size_t b, size_t *c) { + if (SIZE_MAX - a < b) return false; + *c = a + b; + return true; +} + +static size_t saturating_multiply(size_t a, size_t b) { + /* size_t is unsigned, so this is defined behavior even on overflow. */ + size_t ret = a * b; + if (b != 0 && ret / b != a) { + ret = SIZE_MAX; + } + return ret; +} + + +/* Base64 decoding ************************************************************/ + +/* TODO(haberman): make this streaming. */ + +static const signed char b64table[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62/*+*/, -1, -1, -1, 63/*/ */, + 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, + 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, + -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, + 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, + 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1, + -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, + 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/, + 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/, + 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* Returns the table value sign-extended to 32 bits. Knowing that the upper + * bits will be 1 for unrecognized characters makes it easier to check for + * this error condition later (see below). */ +int32_t b64lookup(unsigned char ch) { return b64table[ch]; } + +/* Returns true if the given character is not a valid base64 character or + * padding. */ +bool nonbase64(unsigned char ch) { return b64lookup(ch) == -1 && ch != '='; } + +static bool base64_push(upb_json_parser *p, upb_selector_t sel, const char *ptr, + size_t len) { + const char *limit = ptr + len; + for (; ptr < limit; ptr += 4) { + uint32_t val; + char output[3]; + + if (limit - ptr < 4) { + upb_status_seterrf(p->status, + "Base64 input for bytes field not a multiple of 4: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6 | + b64lookup(ptr[3]); + + /* Test the upper bit; returns true if any of the characters returned -1. */ + if (val & 0x80000000) { + goto otherchar; + } + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + output[2] = val & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 3, NULL); + } + return true; + +otherchar: + if (nonbase64(ptr[0]) || nonbase64(ptr[1]) || nonbase64(ptr[2]) || + nonbase64(ptr[3]) ) { + upb_status_seterrf(p->status, + "Non-base64 characters in bytes field: %s", + upb_fielddef_name(p->top->f)); + return false; + } if (ptr[2] == '=') { + uint32_t val; + char output; + + /* Last group contains only two input bytes, one output byte. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[3] != '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12; + + UPB_ASSERT(!(val & 0x80000000)); + output = val >> 16; + upb_sink_putstring(p->top->sink, sel, &output, 1, NULL); + return true; + } else { + uint32_t val; + char output[2]; + + /* Last group contains only three input bytes, two output bytes. */ + if (ptr[0] == '=' || ptr[1] == '=' || ptr[2] == '=') { + goto badpadding; + } + + val = b64lookup(ptr[0]) << 18 | + b64lookup(ptr[1]) << 12 | + b64lookup(ptr[2]) << 6; + + output[0] = val >> 16; + output[1] = (val >> 8) & 0xff; + upb_sink_putstring(p->top->sink, sel, output, 2, NULL); + return true; + } + +badpadding: + upb_status_seterrf(p->status, + "Incorrect base64 padding for field: %s (%.*s)", + upb_fielddef_name(p->top->f), + 4, ptr); + return false; +} + + +/* Accumulate buffer **********************************************************/ + +/* Functionality for accumulating a buffer. + * + * Some parts of the parser need an entire value as a contiguous string. For + * example, to look up a member name in a hash table, or to turn a string into + * a number, the relevant library routines need the input string to be in + * contiguous memory, even if the value spanned two or more buffers in the + * input. These routines handle that. + * + * In the common case we can just point to the input buffer to get this + * contiguous string and avoid any actual copy. So we optimistically begin + * this way. But there are a few cases where we must instead copy into a + * separate buffer: + * + * 1. The string was not contiguous in the input (it spanned buffers). + * + * 2. The string included escape sequences that need to be interpreted to get + * the true value in a contiguous buffer. */ + +static void assert_accumulate_empty(upb_json_parser *p) { + UPB_ASSERT(p->accumulated == NULL); + UPB_ASSERT(p->accumulated_len == 0); +} + +static void accumulate_clear(upb_json_parser *p) { + p->accumulated = NULL; + p->accumulated_len = 0; +} + +/* Used internally by accumulate_append(). */ +static bool accumulate_realloc(upb_json_parser *p, size_t need) { + void *mem; + size_t old_size = p->accumulate_buf_size; + size_t new_size = UPB_MAX(old_size, 128); + while (new_size < need) { + new_size = saturating_multiply(new_size, 2); + } + + mem = upb_arena_realloc(p->arena, p->accumulate_buf, old_size, new_size); + if (!mem) { + upb_status_seterrmsg(p->status, "Out of memory allocating buffer."); + return false; + } + + p->accumulate_buf = mem; + p->accumulate_buf_size = new_size; + return true; +} + +/* Logically appends the given data to the append buffer. + * If "can_alias" is true, we will try to avoid actually copying, but the buffer + * must be valid until the next accumulate_append() call (if any). */ +static bool accumulate_append(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + size_t need; + + if (!p->accumulated && can_alias) { + p->accumulated = buf; + p->accumulated_len = len; + return true; + } + + if (!checked_add(p->accumulated_len, len, &need)) { + upb_status_seterrmsg(p->status, "Integer overflow."); + return false; + } + + if (need > p->accumulate_buf_size && !accumulate_realloc(p, need)) { + return false; + } + + if (p->accumulated != p->accumulate_buf) { + memcpy(p->accumulate_buf, p->accumulated, p->accumulated_len); + p->accumulated = p->accumulate_buf; + } + + memcpy(p->accumulate_buf + p->accumulated_len, buf, len); + p->accumulated_len += len; + return true; +} + +/* Returns a pointer to the data accumulated since the last accumulate_clear() + * call, and writes the length to *len. This with point either to the input + * buffer or a temporary accumulate buffer. */ +static const char *accumulate_getptr(upb_json_parser *p, size_t *len) { + UPB_ASSERT(p->accumulated); + *len = p->accumulated_len; + return p->accumulated; +} + + +/* Mult-part text data ********************************************************/ + +/* When we have text data in the input, it can often come in multiple segments. + * For example, there may be some raw string data followed by an escape + * sequence. The two segments are processed with different logic. Also buffer + * seams in the input can cause multiple segments. + * + * As we see segments, there are two main cases for how we want to process them: + * + * 1. we want to push the captured input directly to string handlers. + * + * 2. we need to accumulate all the parts into a contiguous buffer for further + * processing (field name lookup, string->number conversion, etc). */ + +/* This is the set of states for p->multipart_state. */ +enum { + /* We are not currently processing multipart data. */ + MULTIPART_INACTIVE = 0, + + /* We are processing multipart data by accumulating it into a contiguous + * buffer. */ + MULTIPART_ACCUMULATE = 1, + + /* We are processing multipart data by pushing each part directly to the + * current string handlers. */ + MULTIPART_PUSHEAGERLY = 2 +}; + +/* Start a multi-part text value where we accumulate the data for processing at + * the end. */ +static void multipart_startaccum(upb_json_parser *p) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_ACCUMULATE; +} + +/* Start a multi-part text value where we immediately push text data to a string + * value with the given selector. */ +static void multipart_start(upb_json_parser *p, upb_selector_t sel) { + assert_accumulate_empty(p); + UPB_ASSERT(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_PUSHEAGERLY; + p->string_selector = sel; +} + +static bool multipart_text(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + switch (p->multipart_state) { + case MULTIPART_INACTIVE: + upb_status_seterrmsg( + p->status, "Internal error: unexpected state MULTIPART_INACTIVE"); + return false; + + case MULTIPART_ACCUMULATE: + if (!accumulate_append(p, buf, len, can_alias)) { + return false; + } + break; + + case MULTIPART_PUSHEAGERLY: { + const upb_bufhandle *handle = can_alias ? p->handle : NULL; + upb_sink_putstring(p->top->sink, p->string_selector, buf, len, handle); + break; + } + } + + return true; +} + +/* Note: this invalidates the accumulate buffer! Call only after reading its + * contents. */ +static void multipart_end(upb_json_parser *p) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_INACTIVE; + accumulate_clear(p); +} + + +/* Input capture **************************************************************/ + +/* Functionality for capturing a region of the input as text. Gracefully + * handles the case where a buffer seam occurs in the middle of the captured + * region. */ + +static void capture_begin(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->multipart_state != MULTIPART_INACTIVE); + UPB_ASSERT(p->capture == NULL); + p->capture = ptr; +} + +static bool capture_end(upb_json_parser *p, const char *ptr) { + UPB_ASSERT(p->capture); + if (multipart_text(p, p->capture, ptr - p->capture, true)) { + p->capture = NULL; + return true; + } else { + return false; + } +} + +/* This is called at the end of each input buffer (ie. when we have hit a + * buffer seam). If we are in the middle of capturing the input, this + * processes the unprocessed capture region. */ +static void capture_suspend(upb_json_parser *p, const char **ptr) { + if (!p->capture) return; + + if (multipart_text(p, p->capture, *ptr - p->capture, false)) { + /* We use this as a signal that we were in the middle of capturing, and + * that capturing should resume at the beginning of the next buffer. + * + * We can't use *ptr here, because we have no guarantee that this pointer + * will be valid when we resume (if the underlying memory is freed, then + * using the pointer at all, even to compare to NULL, is likely undefined + * behavior). */ + p->capture = &suspend_capture; + } else { + /* Need to back up the pointer to the beginning of the capture, since + * we were not able to actually preserve it. */ + *ptr = p->capture; + } +} + +static void capture_resume(upb_json_parser *p, const char *ptr) { + if (p->capture) { + UPB_ASSERT(p->capture == &suspend_capture); + p->capture = ptr; + } +} + + +/* Callbacks from the parser **************************************************/ + +/* These are the functions called directly from the parser itself. + * We define these in the same order as their declarations in the parser. */ + +static char escape_char(char in) { + switch (in) { + case 'r': return '\r'; + case 't': return '\t'; + case 'n': return '\n'; + case 'f': return '\f'; + case 'b': return '\b'; + case '/': return '/'; + case '"': return '"'; + case '\\': return '\\'; + default: + UPB_ASSERT(0); + return 'x'; + } +} + +static bool escape(upb_json_parser *p, const char *ptr) { + char ch = escape_char(*ptr); + return multipart_text(p, &ch, 1, false); +} + +static void start_hex(upb_json_parser *p) { + p->digit = 0; +} + +static void hexdigit(upb_json_parser *p, const char *ptr) { + char ch = *ptr; + + p->digit <<= 4; + + if (ch >= '0' && ch <= '9') { + p->digit += (ch - '0'); + } else if (ch >= 'a' && ch <= 'f') { + p->digit += ((ch - 'a') + 10); + } else { + UPB_ASSERT(ch >= 'A' && ch <= 'F'); + p->digit += ((ch - 'A') + 10); + } +} + +static bool end_hex(upb_json_parser *p) { + uint32_t codepoint = p->digit; + + /* emit the codepoint as UTF-8. */ + char utf8[3]; /* support \u0000 -- \uFFFF -- need only three bytes. */ + int length = 0; + if (codepoint <= 0x7F) { + utf8[0] = codepoint; + length = 1; + } else if (codepoint <= 0x07FF) { + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x1F) | 0xC0; + length = 2; + } else /* codepoint <= 0xFFFF */ { + utf8[2] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x0F) | 0xE0; + length = 3; + } + /* TODO(haberman): Handle high surrogates: if codepoint is a high surrogate + * we have to wait for the next escape to get the full code point). */ + + return multipart_text(p, utf8, length, false); +} + +static void start_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_number(upb_json_parser *p, const char *ptr) { + if (is_top_level(p)) { + if (is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NUMBERVALUE); + } else { + return false; + } + } else if (does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NUMBERVALUE); + } + + multipart_startaccum(p); + capture_begin(p, ptr); + return true; +} + +static bool parse_number(upb_json_parser *p, bool is_quoted); + +static bool end_number_nontop(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + return parse_number(p, false); +} + +static bool end_number(upb_json_parser *p, const char *ptr) { + if (!end_number_nontop(p, ptr)) { + return false; + } + + if (does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +/* |buf| is NULL-terminated. |buf| itself will never include quotes; + * |is_quoted| tells us whether this text originally appeared inside quotes. */ +static bool parse_number_from_buffer(upb_json_parser *p, const char *buf, + bool is_quoted) { + size_t len = strlen(buf); + const char *bufend = buf + len; + char *end; + upb_fieldtype_t type = upb_fielddef_type(p->top->f); + double val; + double dummy; + double inf = UPB_INFINITY; + + errno = 0; + + if (len == 0 || buf[0] == ' ') { + return false; + } + + /* For integer types, first try parsing with integer-specific routines. + * If these succeed, they will be more accurate for int64/uint64 than + * strtod(). + */ + switch (type) { + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else if (val > INT32_MAX || val < INT32_MIN) { + return false; + } else { + upb_sink_putint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT32: { + unsigned long val = strtoul(buf, &end, 0); + if (end != bufend) { + break; + } else if (val > UINT32_MAX || errno == ERANGE) { + return false; + } else { + upb_sink_putuint32(p->top->sink, parser_getsel(p), val); + return true; + } + } + /* XXX: We can't handle [u]int64 properly on 32-bit machines because + * strto[u]ll isn't in C89. */ + case UPB_TYPE_INT64: { + long val = strtol(buf, &end, 0); + if (errno == ERANGE || end != bufend) { + break; + } else { + upb_sink_putint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + case UPB_TYPE_UINT64: { + unsigned long val = strtoul(p->accumulated, &end, 0); + if (end != bufend) { + break; + } else if (errno == ERANGE) { + return false; + } else { + upb_sink_putuint64(p->top->sink, parser_getsel(p), val); + return true; + } + } + default: + break; + } + + if (type != UPB_TYPE_DOUBLE && type != UPB_TYPE_FLOAT && is_quoted) { + /* Quoted numbers for integer types are not allowed to be in double form. */ + return false; + } + + if (len == strlen("Infinity") && strcmp(buf, "Infinity") == 0) { + /* C89 does not have an INFINITY macro. */ + val = inf; + } else if (len == strlen("-Infinity") && strcmp(buf, "-Infinity") == 0) { + val = -inf; + } else { + val = strtod(buf, &end); + if (errno == ERANGE || end != bufend) { + return false; + } + } + + switch (type) { +#define CASE(capitaltype, smalltype, ctype, min, max) \ + case UPB_TYPE_ ## capitaltype: { \ + if (modf(val, &dummy) != 0 || val > max || val < min) { \ + return false; \ + } else { \ + upb_sink_put ## smalltype(p->top->sink, parser_getsel(p), \ + (ctype)val); \ + return true; \ + } \ + break; \ + } + case UPB_TYPE_ENUM: + CASE(INT32, int32, int32_t, INT32_MIN, INT32_MAX); + CASE(INT64, int64, int64_t, INT64_MIN, INT64_MAX); + CASE(UINT32, uint32, uint32_t, 0, UINT32_MAX); + CASE(UINT64, uint64, uint64_t, 0, UINT64_MAX); +#undef CASE + + case UPB_TYPE_DOUBLE: + upb_sink_putdouble(p->top->sink, parser_getsel(p), val); + return true; + case UPB_TYPE_FLOAT: + if ((val > FLT_MAX || val < -FLT_MAX) && val != inf && val != -inf) { + return false; + } else { + upb_sink_putfloat(p->top->sink, parser_getsel(p), val); + return true; + } + default: + return false; + } +} + +static bool parse_number(upb_json_parser *p, bool is_quoted) { + size_t len; + const char *buf; + + /* strtol() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + if (!multipart_text(p, "\0", 1, false)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (parse_number_from_buffer(p, buf, is_quoted)) { + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "error parsing number: %s", buf); + multipart_end(p); + return false; + } +} + +static bool parser_putbool(upb_json_parser *p, bool val) { + bool ok; + + if (p->top->f == NULL) { + return true; + } + + if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { + upb_status_seterrf(p->status, + "Boolean value specified for non-bool field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + ok = upb_sink_putbool(p->top->sink, parser_getsel(p), val); + UPB_ASSERT(ok); + + return true; +} + +static bool end_bool(upb_json_parser *p, bool val) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_BOOLVALUE); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_BOOLVALUE)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_BOOLVALUE); + } + + if (p->top->is_unknown_field) { + return true; + } + + if (!parser_putbool(p, val)) { + return false; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_BOOLVALUE)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static bool end_null(upb_json_parser *p) { + const char *zero_ptr = "0"; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_NULLVALUE); + } else { + return true; + } + + /* Fill null_value field. */ + multipart_startaccum(p); + capture_begin(p, zero_ptr); + capture_end(p, zero_ptr + 1); + parse_number(p, false); + + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + + return true; +} + +static bool start_any_stringval(upb_json_parser *p) { + multipart_startaccum(p); + return true; +} + +static bool start_stringval(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_string_wrapper_object(p) || + is_number_wrapper_object(p)) { + start_wrapper_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + start_fieldmask_object(p); + return true; + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + start_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRINGVALUE); + } else { + return false; + } + } else if (does_string_wrapper_start(p) || + does_number_wrapper_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_wrapper_object(p); + } else if (does_fieldmask_start(p)) { + if (!start_subobject(p)) { + return false; + } + start_fieldmask_object(p); + return true; + } else if (is_wellknown_field(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_field(p, UPB_WELLKNOWN_DURATION)) { + if (!start_subobject(p)) { + return false; + } + start_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) { + return false; + } + start_value_object(p, VALUE_STRINGVALUE); + } + + if (p->top->f == NULL) { + multipart_startaccum(p); + return true; + } + + if (p->top->is_any) { + return start_any_stringval(p); + } + + if (upb_fielddef_isstring(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + if (upb_fielddef_type(p->top->f) == UPB_TYPE_STRING) { + /* For STRING fields we push data directly to the handlers as it is + * parsed. We don't do this yet for BYTES fields, because our base64 + * decoder is not streaming. + * + * TODO(haberman): make base64 decoding streaming also. */ + multipart_start(p, getsel_for_handlertype(p, UPB_HANDLER_STRING)); + return true; + } else { + multipart_startaccum(p); + return true; + } + } else if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL && + upb_fielddef_type(p->top->f) != UPB_TYPE_MESSAGE) { + /* No need to push a frame -- numeric values in quotes remain in the + * current parser frame. These values must accmulate so we can convert + * them all at once at the end. */ + multipart_startaccum(p); + return true; + } else { + upb_status_seterrf(p->status, + "String specified for bool or submessage field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool end_any_stringval(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Set type_url */ + upb_selector_t sel; + upb_jsonparser_frame *inner; + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + multipart_end(p); + + /* Resolve type url */ + if (strncmp(buf, "type.googleapis.com/", 20) == 0 && len > 20) { + const upb_msgdef *payload_type = NULL; + buf += 20; + len -= 20; + + payload_type = upb_symtab_lookupmsg2(p->symtab, buf, len); + if (payload_type == NULL) { + upb_status_seterrf( + p->status, "Cannot find packed type: %.*s\n", (int)len, buf); + return false; + } + + json_parser_any_frame_set_payload_type(p, p->top->any_frame, payload_type); + + return true; + } else { + upb_status_seterrf( + p->status, "Invalid type url: %.*s\n", (int)len, buf); + return false; + } +} + +static bool end_stringval_nontop(upb_json_parser *p) { + bool ok = true; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION)) { + multipart_end(p); + return true; + } + + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_stringval(p); + } + + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_BYTES: + if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + /* Fall through. */ + + case UPB_TYPE_STRING: { + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + break; + } + + case UPB_TYPE_ENUM: { + /* Resolve enum symbolic name to integer value. */ + const upb_enumdef *enumdef = upb_fielddef_enumsubdef(p->top->f); + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + int32_t int_val = 0; + ok = upb_enumdef_ntoi(enumdef, buf, len, &int_val); + + if (ok) { + upb_selector_t sel = parser_getsel(p); + upb_sink_putint32(p->top->sink, sel, int_val); + } else { + upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf); + } + + break; + } + + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_FLOAT: + ok = parse_number(p, true); + break; + + default: + UPB_ASSERT(false); + upb_status_seterrmsg(p->status, "Internal error in JSON decoder"); + ok = false; + break; + } + + multipart_end(p); + + return ok; +} + +static bool end_stringval(upb_json_parser *p) { + /* FieldMask's stringvals have been ended when handling them. Only need to + * close FieldMask here.*/ + if (does_fieldmask_end(p)) { + end_fieldmask_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (!end_stringval_nontop(p)) { + return false; + } + + if (does_string_wrapper_end(p) || + does_number_wrapper_end(p)) { + end_wrapper_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_TIMESTAMP) || + is_wellknown_msg(p, UPB_WELLKNOWN_DURATION) || + is_wellknown_msg(p, UPB_WELLKNOWN_FIELDMASK)) { + end_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + return true; + } + + return true; +} + +static void start_duration_base(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_duration_base(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char seconds_buf[14]; + char nanos_buf[12]; + char *end; + int64_t seconds = 0; + int32_t nanos = 0; + double val = 0.0; + const char *seconds_membername = "seconds"; + const char *nanos_membername = "nanos"; + size_t fraction_start; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + memset(seconds_buf, 0, 14); + memset(nanos_buf, 0, 12); + + /* Find out base end. The maximus duration is 315576000000, which cannot be + * represented by double without losing precision. Thus, we need to handle + * fraction and base separately. */ + for (fraction_start = 0; fraction_start < len && buf[fraction_start] != '.'; + fraction_start++); + + /* Parse base */ + memcpy(seconds_buf, buf, fraction_start); + seconds = strtol(seconds_buf, &end, 10); + if (errno == ERANGE || end != seconds_buf + fraction_start) { + upb_status_seterrf(p->status, "error parsing duration: %s", + seconds_buf); + return false; + } + + if (seconds > 315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "maximum acceptable value is " + "315576000000"); + return false; + } + + if (seconds < -315576000000) { + upb_status_seterrf(p->status, "error parsing duration: " + "minimum acceptable value is " + "-315576000000"); + return false; + } + + /* Parse fraction */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf + fraction_start, len - fraction_start); + val = strtod(nanos_buf, &end); + if (errno == ERANGE || end != nanos_buf + len - fraction_start + 1) { + upb_status_seterrf(p->status, "error parsing duration: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + if (seconds < 0) nanos = -nanos; + + /* Clean up buffer */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous arena */ + multipart_startaccum(p); + + return true; +} + +static int parse_timestamp_number(upb_json_parser *p) { + size_t len; + const char *buf; + int val; + + /* atoi() and friends unfortunately do not support specifying the length of + * the input string, so we need to force a copy into a NULL-terminated buffer. */ + multipart_text(p, "\0", 1, false); + + buf = accumulate_getptr(p, &len); + val = atoi(buf); + multipart_end(p); + multipart_startaccum(p); + + return val; +} + +static void start_year(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_year(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_year = parse_timestamp_number(p) - 1900; + return true; +} + +static void start_month(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_month(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mon = parse_timestamp_number(p) - 1; + return true; +} + +static void start_day(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_day(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_mday = parse_timestamp_number(p); + return true; +} + +static void start_hour(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_hour(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_hour = parse_timestamp_number(p); + return true; +} + +static void start_minute(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_minute(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_min = parse_timestamp_number(p); + return true; +} + +static void start_second(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_second(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + p->tm.tm_sec = parse_timestamp_number(p); + return true; +} + +static void start_timestamp_base(upb_json_parser *p) { + memset(&p->tm, 0, sizeof(struct tm)); +} + +static void start_timestamp_fraction(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_timestamp_fraction(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + char nanos_buf[12]; + char *end; + double val = 0.0; + int32_t nanos; + const char *nanos_membername = "nanos"; + + memset(nanos_buf, 0, 12); + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (len > 10) { + upb_status_seterrf(p->status, + "error parsing timestamp: at most 9-digit fraction."); + return false; + } + + /* Parse nanos */ + nanos_buf[0] = '0'; + memcpy(nanos_buf + 1, buf, len); + val = strtod(nanos_buf, &end); + + if (errno == ERANGE || end != nanos_buf + len + 1) { + upb_status_seterrf(p->status, "error parsing timestamp nanos: %s", + nanos_buf); + return false; + } + + nanos = val * 1000000000; + + /* Clean up previous environment */ + multipart_end(p); + + /* Set nanos */ + start_member(p); + capture_begin(p, nanos_membername); + capture_end(p, nanos_membername + 5); + end_membername(p); + upb_sink_putint32(p->top->sink, parser_getsel(p), nanos); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_timestamp_zone(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +#define EPOCH_YEAR 1970 +#define TM_YEAR_BASE 1900 + +static bool isleap(int year) { + return (year % 4) == 0 && (year % 100 != 0 || (year % 400) == 0); +} + +const unsigned short int __mon_yday[2][13] = { + /* Normal years. */ + { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, + /* Leap years. */ + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } +}; + +int64_t epoch(int year, int yday, int hour, int min, int sec) { + int64_t years = year - EPOCH_YEAR; + + int64_t leap_days = years / 4 - years / 100 + years / 400; + + int64_t days = years * 365 + yday + leap_days; + int64_t hours = days * 24 + hour; + int64_t mins = hours * 60 + min; + int64_t secs = mins * 60 + sec; + return secs; +} + + +static int64_t upb_mktime(const struct tm *tp) { + int sec = tp->tm_sec; + int min = tp->tm_min; + int hour = tp->tm_hour; + int mday = tp->tm_mday; + int mon = tp->tm_mon; + int year = tp->tm_year + TM_YEAR_BASE; + + /* Calculate day of year from year, month, and day of month. */ + int mon_yday = ((__mon_yday[isleap(year)][mon]) - 1); + int yday = mon_yday + mday; + + return epoch(year, yday, hour, min, sec); +} + +static bool end_timestamp_zone(upb_json_parser *p, const char *ptr) { + size_t len; + const char *buf; + int hours; + int64_t seconds; + const char *seconds_membername = "seconds"; + + if (!capture_end(p, ptr)) { + return false; + } + + buf = accumulate_getptr(p, &len); + + if (buf[0] != 'Z') { + if (sscanf(buf + 1, "%2d:00", &hours) != 1) { + upb_status_seterrf(p->status, "error parsing timestamp offset"); + return false; + } + + if (buf[0] == '+') { + hours = -hours; + } + + p->tm.tm_hour += hours; + } + + /* Normalize tm */ + seconds = upb_mktime(&p->tm); + + /* Check timestamp boundary */ + if (seconds < -62135596800) { + upb_status_seterrf(p->status, "error parsing timestamp: " + "minimum acceptable value is " + "0001-01-01T00:00:00Z"); + return false; + } + + /* Clean up previous environment */ + multipart_end(p); + + /* Set seconds */ + start_member(p); + capture_begin(p, seconds_membername); + capture_end(p, seconds_membername + 7); + end_membername(p); + upb_sink_putint64(p->top->sink, parser_getsel(p), seconds); + end_member(p); + + /* Continue previous environment */ + multipart_startaccum(p); + + return true; +} + +static void start_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_fieldmask_path_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + +static bool start_fieldmask_path(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (!check_stack(p)) return false; + + /* Start a new parser frame: parser frames correspond one-to-one with + * handler frames, and string events occur in a sub-frame. */ + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + multipart_startaccum(p); + return true; +} + +static bool lower_camel_push( + upb_json_parser *p, upb_selector_t sel, const char *ptr, size_t len) { + const char *limit = ptr + len; + bool first = true; + for (;ptr < limit; ptr++) { + if (*ptr >= 'A' && *ptr <= 'Z' && !first) { + char lower = tolower(*ptr); + upb_sink_putstring(p->top->sink, sel, "_", 1, NULL); + upb_sink_putstring(p->top->sink, sel, &lower, 1, NULL); + } else { + upb_sink_putstring(p->top->sink, sel, ptr, 1, NULL); + } + first = false; + } + return true; +} + +static bool end_fieldmask_path(upb_json_parser *p) { + upb_selector_t sel; + + if (!lower_camel_push( + p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(p->top->sink, sel); + p->top--; + + multipart_end(p); + return true; +} + +static void start_member(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + multipart_startaccum(p); +} + +/* Helper: invoked during parse_mapentry() to emit the mapentry message's key + * field based on the current contents of the accumulate buffer. */ +static bool parse_mapentry_key(upb_json_parser *p) { + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + /* Emit the key field. We do a bit of ad-hoc parsing here because the + * parser state machine has already decided that this is a string field + * name, and we are reinterpreting it as some arbitrary key type. In + * particular, integer and bool keys are quoted, so we need to parse the + * quoted string contents here. */ + + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_KEY); + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no key"); + return false; + } + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + /* Invoke end_number. The accum buffer has the number's text already. */ + if (!parse_number(p, true)) { + return false; + } + break; + case UPB_TYPE_BOOL: + if (len == 4 && !strncmp(buf, "true", 4)) { + if (!parser_putbool(p, true)) { + return false; + } + } else if (len == 5 && !strncmp(buf, "false", 5)) { + if (!parser_putbool(p, false)) { + return false; + } + } else { + upb_status_seterrmsg(p->status, + "Map bool key not 'true' or 'false'"); + return false; + } + multipart_end(p); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + upb_sink subsink; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, len, &subsink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(subsink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(subsink, sel); + multipart_end(p); + break; + } + default: + upb_status_seterrmsg(p->status, "Invalid field type for map key"); + return false; + } + + return true; +} + +/* Helper: emit one map entry (as a submessage in the map field sequence). This + * is invoked from end_membername(), at the end of the map entry's key string, + * with the map key in the accumulate buffer. It parses the key from that + * buffer, emits the handler calls to start the mapentry submessage (setting up + * its subframe in the process), and sets up state in the subframe so that the + * value parser (invoked next) will emit the mapentry's value field and then + * end the mapentry message. */ + +static bool handle_mapentry(upb_json_parser *p) { + const upb_fielddef *mapfield; + const upb_msgdef *mapentrymsg; + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Map entry: p->top->sink is the seq frame, so we need to start a frame + * for the mapentry itself, and then set |f| in that frame so that the map + * value field is parsed, and also set a flag to end the frame after the + * map-entry value is parsed. */ + if (!check_stack(p)) return false; + + mapfield = p->top->mapfield; + mapentrymsg = upb_fielddef_msgsubdef(mapfield); + + inner = start_jsonparser_frame(p); + p->top->f = mapfield; + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = mapentrymsg; + inner->mapfield = mapfield; + + /* Don't set this to true *yet* -- we reuse parsing handlers below to push + * the key field value to the sink, and these handlers will pop the frame + * if they see is_mapentry (when invoked by the parser state machine, they + * would have just seen the map-entry value, not key). */ + inner->is_mapentry = false; + p->top = inner; + + /* send STARTMSG in submsg frame. */ + upb_sink_startmsg(p->top->sink); + + parse_mapentry_key(p); + + /* Set up the value field to receive the map-entry value. */ + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_VALUE); + p->top->is_mapentry = true; /* set up to pop frame after value is parsed. */ + p->top->mapfield = mapfield; + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no value"); + return false; + } + + return true; +} + +static bool end_membername(upb_json_parser *p) { + UPB_ASSERT(!p->top->f); + + if (!p->top->m) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } + + if (p->top->is_any) { + return end_any_membername(p); + } else if (p->top->is_map) { + return handle_mapentry(p); + } else { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (upb_strtable_lookup2(p->top->name_table, buf, len, &v)) { + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + + return true; + } else if (p->ignore_json_unknown) { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } else { + upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + return false; + } + } +} + +static bool end_any_membername(upb_json_parser *p) { + size_t len; + const char *buf = accumulate_getptr(p, &len); + upb_value v; + + if (len == 5 && strncmp(buf, "@type", len) == 0) { + upb_strtable_lookup2(p->top->name_table, "type_url", 8, &v); + p->top->f = upb_value_getconstptr(v); + multipart_end(p); + return true; + } else { + p->top->is_unknown_field = true; + multipart_end(p); + return true; + } +} + +static void end_member(upb_json_parser *p) { + /* If we just parsed a map-entry value, end that frame too. */ + if (p->top->is_mapentry) { + upb_selector_t sel; + bool ok; + const upb_fielddef *mapfield; + + UPB_ASSERT(p->top > p->stack); + /* send ENDMSG on submsg. */ + upb_sink_endmsg(p->top->sink, p->status); + mapfield = p->top->mapfield; + + /* send ENDSUBMSG in repeated-field-of-mapentries frame. */ + p->top--; + ok = upb_handlers_getselector(mapfield, UPB_HANDLER_ENDSUBMSG, &sel); + UPB_ASSERT(ok); + upb_sink_endsubmsg(p->top->sink, sel); + } + + p->top->f = NULL; + p->top->is_unknown_field = false; +} + +static void start_any_member(upb_json_parser *p, const char *ptr) { + start_member(p); + json_parser_any_frame_set_after_type_url_start_once(p->top->any_frame, ptr); +} + +static void end_any_member(upb_json_parser *p, const char *ptr) { + json_parser_any_frame_set_before_type_url_end(p->top->any_frame, ptr); + end_member(p); +} + +static bool start_subobject(upb_json_parser *p) { + if (p->top->is_unknown_field) { + if (!check_stack(p)) return false; + + p->top = start_jsonparser_frame(p); + return true; + } + + if (upb_fielddef_ismap(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a map. Start a new parser frame in a repeated-field + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->mapfield = p->top->f; + inner->is_map = true; + p->top = inner; + + return true; + } else if (upb_fielddef_issubmsg(p->top->f)) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + /* Beginning of a subobject. Start a new parser frame in the submsg + * context. */ + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + set_name_table(p, inner); + p->top = inner; + + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + + return true; + } else { + upb_status_seterrf(p->status, + "Object specified for non-message/group field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool start_subobject_full(upb_json_parser *p) { + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + start_structvalue_object(p); + } else { + return true; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_STRUCT)) { + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_STRUCTVALUE); + if (!start_subobject(p)) return false; + start_structvalue_object(p); + } + + return start_subobject(p); +} + +static void end_subobject(upb_json_parser *p) { + if (is_top_level(p)) { + return; + } + + if (p->top->is_map) { + upb_selector_t sel; + p->top--; + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + } else { + upb_selector_t sel; + bool is_unknown = p->top->m == NULL; + p->top--; + if (!is_unknown) { + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(p->top->sink, sel); + } + } +} + +static void end_subobject_full(upb_json_parser *p) { + end_subobject(p); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_STRUCT)) { + end_structvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static bool start_array(upb_json_parser *p) { + upb_jsonparser_frame *inner; + upb_selector_t sel; + + if (is_top_level(p)) { + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + start_listvalue_object(p); + } else { + return false; + } + } else if (is_wellknown_field(p, UPB_WELLKNOWN_LISTVALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } else if (is_wellknown_field(p, UPB_WELLKNOWN_VALUE) && + (!upb_fielddef_isseq(p->top->f) || + p->top->is_repeated)) { + if (!start_subobject(p)) return false; + start_value_object(p, VALUE_LISTVALUE); + if (!start_subobject(p)) return false; + start_listvalue_object(p); + } + + if (p->top->is_unknown_field) { + inner = start_jsonparser_frame(p); + inner->is_unknown_field = true; + p->top = inner; + + return true; + } + + if (!upb_fielddef_isseq(p->top->f)) { + upb_status_seterrf(p->status, + "Array specified for non-repeated field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + if (!check_stack(p)) return false; + + inner = start_jsonparser_frame(p); + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(p->top->sink, sel, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + inner->is_repeated = true; + p->top = inner; + + return true; +} + +static void end_array(upb_json_parser *p) { + upb_selector_t sel; + + UPB_ASSERT(p->top > p->stack); + + p->top--; + + if (p->top->is_unknown_field) { + return; + } + + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(p->top->sink, sel); + + if (is_wellknown_msg(p, UPB_WELLKNOWN_LISTVALUE)) { + end_listvalue_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } + + if (is_wellknown_msg(p, UPB_WELLKNOWN_VALUE)) { + end_value_object(p); + if (!is_top_level(p)) { + end_subobject(p); + } + } +} + +static void start_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_startmsg(p->top->sink); + } +} + +static void end_object(upb_json_parser *p) { + if (!p->top->is_map && p->top->m != NULL) { + upb_sink_endmsg(p->top->sink, p->status); + } +} + +static void start_any_object(upb_json_parser *p, const char *ptr) { + start_object(p); + p->top->any_frame->before_type_url_start = ptr; + p->top->any_frame->before_type_url_end = ptr; +} + +static bool end_any_object(upb_json_parser *p, const char *ptr) { + const char *value_membername = "value"; + bool is_well_known_packed = false; + const char *packed_end = ptr + 1; + upb_selector_t sel; + upb_jsonparser_frame *inner; + + if (json_parser_any_frame_has_value(p->top->any_frame) && + !json_parser_any_frame_has_type_url(p->top->any_frame)) { + upb_status_seterrmsg(p->status, "No valid type url"); + return false; + } + + /* Well known types data is represented as value field. */ + if (upb_msgdef_wellknowntype(p->top->any_frame->parser->top->m) != + UPB_WELLKNOWN_UNSPECIFIED) { + is_well_known_packed = true; + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + p->top->any_frame->before_type_url_start = + memchr(p->top->any_frame->before_type_url_start, ':', + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start); + if (p->top->any_frame->before_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "invalid data for well known type."); + return false; + } + p->top->any_frame->before_type_url_start++; + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + p->top->any_frame->after_type_url_start = + memchr(p->top->any_frame->after_type_url_start, ':', + (ptr + 1) - + p->top->any_frame->after_type_url_start); + if (p->top->any_frame->after_type_url_start == NULL) { + upb_status_seterrmsg(p->status, "Invalid data for well known type."); + return false; + } + p->top->any_frame->after_type_url_start++; + packed_end = ptr; + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->before_type_url_start, + p->top->any_frame->before_type_url_end - + p->top->any_frame->before_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "{", 1, NULL)) { + return false; + } + } + } + + if (json_parser_any_frame_has_value_before_type_url(p->top->any_frame) && + json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, ",", 1, NULL)) { + return false; + } + } + + if (json_parser_any_frame_has_value_after_type_url(p->top->any_frame)) { + if (!parse(p->top->any_frame->parser, NULL, + p->top->any_frame->after_type_url_start, + packed_end - p->top->any_frame->after_type_url_start, NULL)) { + return false; + } + } else { + if (!is_well_known_packed) { + if (!parse(p->top->any_frame->parser, NULL, "}", 1, NULL)) { + return false; + } + } + } + + if (!end(p->top->any_frame->parser, NULL)) { + return false; + } + + p->top->is_any = false; + + /* Set value */ + start_member(p); + capture_begin(p, value_membername); + capture_end(p, value_membername + 5); + end_membername(p); + + if (!check_stack(p)) return false; + inner = p->top + 1; + + sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(p->top->sink, sel, 0, &inner->sink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(inner->sink, sel, p->top->any_frame->stringsink.ptr, + p->top->any_frame->stringsink.len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(inner->sink, sel); + + end_member(p); + + end_object(p); + + /* Deallocate any parse frame. */ + json_parser_any_frame_free(p->top->any_frame); + + return true; +} + +static bool is_string_wrapper(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_STRINGVALUE || + type == UPB_WELLKNOWN_BYTESVALUE; +} + +static bool is_fieldmask(const upb_msgdef *m) { + upb_wellknowntype_t type = upb_msgdef_wellknowntype(m); + return type == UPB_WELLKNOWN_FIELDMASK; +} + +static void start_fieldmask_object(upb_json_parser *p) { + const char *membername = "paths"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); + + start_array(p); +} + +static void end_fieldmask_object(upb_json_parser *p) { + end_array(p); + end_member(p); + end_object(p); +} + +static void start_wrapper_object(upb_json_parser *p) { + const char *membername = "value"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + 5); + end_membername(p); +} + +static void end_wrapper_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_value_object(upb_json_parser *p, int value_type) { + const char *nullmember = "null_value"; + const char *numbermember = "number_value"; + const char *stringmember = "string_value"; + const char *boolmember = "bool_value"; + const char *structmember = "struct_value"; + const char *listmember = "list_value"; + const char *membername = ""; + + switch (value_type) { + case VALUE_NULLVALUE: + membername = nullmember; + break; + case VALUE_NUMBERVALUE: + membername = numbermember; + break; + case VALUE_STRINGVALUE: + membername = stringmember; + break; + case VALUE_BOOLVALUE: + membername = boolmember; + break; + case VALUE_STRUCTVALUE: + membername = structmember; + break; + case VALUE_LISTVALUE: + membername = listmember; + break; + } + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_value_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_listvalue_object(upb_json_parser *p) { + const char *membername = "values"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_listvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static void start_structvalue_object(upb_json_parser *p) { + const char *membername = "fields"; + + start_object(p); + + /* Set up context for parsing value */ + start_member(p); + capture_begin(p, membername); + capture_end(p, membername + strlen(membername)); + end_membername(p); +} + +static void end_structvalue_object(upb_json_parser *p) { + end_member(p); + end_object(p); +} + +static bool is_top_level(upb_json_parser *p) { + return p->top == p->stack && p->top->f == NULL && !p->top->is_unknown_field; +} + +static bool is_wellknown_msg(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->m != NULL && upb_msgdef_wellknowntype(p->top->m) == type; +} + +static bool is_wellknown_field(upb_json_parser *p, upb_wellknowntype_t type) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + (upb_msgdef_wellknowntype(upb_fielddef_msgsubdef(p->top->f)) + == type); +} + +static bool does_number_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + upb_msgdef_isnumberwrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_number_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool is_number_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && upb_msgdef_isnumberwrapper(p->top->m); +} + +static bool does_string_wrapper_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_string_wrapper(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_string_wrapper_end(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool is_string_wrapper_object(upb_json_parser *p) { + return p->top->m != NULL && is_string_wrapper(p->top->m); +} + +static bool does_fieldmask_start(upb_json_parser *p) { + return p->top->f != NULL && + upb_fielddef_issubmsg(p->top->f) && + is_fieldmask(upb_fielddef_msgsubdef(p->top->f)); +} + +static bool does_fieldmask_end(upb_json_parser *p) { + return p->top->m != NULL && is_fieldmask(p->top->m); +} + +#define CHECK_RETURN_TOP(x) if (!(x)) goto error + + +/* The actual parser **********************************************************/ + +/* What follows is the Ragel parser itself. The language is specified in Ragel + * and the actions call our C functions above. + * + * Ragel has an extensive set of functionality, and we use only a small part of + * it. There are many action types but we only use a few: + * + * ">" -- transition into a machine + * "%" -- transition out of a machine + * "@" -- transition into a final state of a machine. + * + * "@" transitions are tricky because a machine can transition into a final + * state repeatedly. But in some cases we know this can't happen, for example + * a string which is delimited by a final '"' can only transition into its + * final state once, when the closing '"' is seen. */ + +%%{ + machine json; + + ws = space*; + + integer = "0" | /[1-9]/ /[0-9]/*; + decimal = "." /[0-9]/+; + exponent = /[eE]/ /[+\-]/? /[0-9]/+; + + number_machine := + ("-"? integer decimal? exponent?) + %/{ fhold; fret; } + <: any + >{ fhold; fret; } + ; + number = /[0-9\-]/ >{ fhold; fcall number_machine; }; + + text = + /[^\\"]/+ + >{ start_text(parser, p); } + %{ CHECK_RETURN_TOP(end_text(parser, p)); } + ; + + unicode_char = + "\\u" + /[0-9A-Fa-f]/{4} + >{ start_hex(parser); } + ${ hexdigit(parser, p); } + %{ CHECK_RETURN_TOP(end_hex(parser)); } + ; + + escape_char = + "\\" + /[rtbfn"\/\\]/ + >{ CHECK_RETURN_TOP(escape(parser, p)); } + ; + + string_machine := + (text | unicode_char | escape_char)** + '"' + @{ fhold; fret; } + ; + + year = + (digit digit digit digit) + >{ start_year(parser, p); } + %{ CHECK_RETURN_TOP(end_year(parser, p)); } + ; + month = + (digit digit) + >{ start_month(parser, p); } + %{ CHECK_RETURN_TOP(end_month(parser, p)); } + ; + day = + (digit digit) + >{ start_day(parser, p); } + %{ CHECK_RETURN_TOP(end_day(parser, p)); } + ; + hour = + (digit digit) + >{ start_hour(parser, p); } + %{ CHECK_RETURN_TOP(end_hour(parser, p)); } + ; + minute = + (digit digit) + >{ start_minute(parser, p); } + %{ CHECK_RETURN_TOP(end_minute(parser, p)); } + ; + second = + (digit digit) + >{ start_second(parser, p); } + %{ CHECK_RETURN_TOP(end_second(parser, p)); } + ; + + duration_machine := + ("-"? integer decimal?) + >{ start_duration_base(parser, p); } + %{ CHECK_RETURN_TOP(end_duration_base(parser, p)); } + 's"' + @{ fhold; fret; } + ; + + timestamp_machine := + (year "-" month "-" day "T" hour ":" minute ":" second) + >{ start_timestamp_base(parser); } + ("." digit+)? + >{ start_timestamp_fraction(parser, p); } + %{ CHECK_RETURN_TOP(end_timestamp_fraction(parser, p)); } + ([+\-] digit digit ":00" | "Z") + >{ start_timestamp_zone(parser, p); } + %{ CHECK_RETURN_TOP(end_timestamp_zone(parser, p)); } + '"' + @{ fhold; fret; } + ; + + fieldmask_path_text = + /[^",]/+ + >{ start_fieldmask_path_text(parser, p); } + %{ end_fieldmask_path_text(parser, p); } + ; + + fieldmask_path = + fieldmask_path_text + >{ start_fieldmask_path(parser); } + %{ end_fieldmask_path(parser); } + ; + + fieldmask_machine := + (fieldmask_path ("," fieldmask_path)*)? + '"' + @{ fhold; fret; } + ; + + string = + '"' + @{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { + fcall timestamp_machine; + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { + fcall duration_machine; + } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_FIELDMASK)) { + fcall fieldmask_machine; + } else { + fcall string_machine; + } + } + '"'; + + value2 = ^(space | "]" | "}") >{ fhold; fcall value_machine; } ; + + member = + ws + string + >{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_member(parser, p); + } else { + start_member(parser); + } + } + @{ CHECK_RETURN_TOP(end_membername(parser)); } + ws ":" ws + value2 + %{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + end_any_member(parser, p); + } else { + end_member(parser); + } + } + ws; + + object = + ("{" ws) + >{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + start_any_object(parser, p); + } else { + start_object(parser); + } + } + (member ("," member)*)? + "}" + >{ + if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) { + CHECK_RETURN_TOP(end_any_object(parser, p)); + } else { + end_object(parser); + } + } + ; + + element = ws value2 ws; + array = + "[" + >{ CHECK_RETURN_TOP(start_array(parser)); } + ws + (element ("," element)*)? + "]" + >{ end_array(parser); } + ; + + value = + number + >{ CHECK_RETURN_TOP(start_number(parser, p)); } + %{ CHECK_RETURN_TOP(end_number(parser, p)); } + | string + >{ CHECK_RETURN_TOP(start_stringval(parser)); } + @{ CHECK_RETURN_TOP(end_stringval(parser)); } + | "true" + %{ CHECK_RETURN_TOP(end_bool(parser, true)); } + | "false" + %{ CHECK_RETURN_TOP(end_bool(parser, false)); } + | "null" + %{ CHECK_RETURN_TOP(end_null(parser)); } + | object + >{ CHECK_RETURN_TOP(start_subobject_full(parser)); } + %{ end_subobject_full(parser); } + | array; + + value_machine := + value + <: any >{ fhold; fret; } ; + + main := ws value ws; +}%% + +%% write data noerror nofinal; + +size_t parse(void *closure, const void *hd, const char *buf, size_t size, + const upb_bufhandle *handle) { + upb_json_parser *parser = closure; + + /* Variables used by Ragel's generated code. */ + int cs = parser->current_state; + int *stack = parser->parser_stack; + int top = parser->parser_top; + + const char *p = buf; + const char *pe = buf + size; + const char *eof = &eof_ch; + + parser->handle = handle; + + UPB_UNUSED(hd); + UPB_UNUSED(handle); + + capture_resume(parser, buf); + + %% write exec; + + if (p != pe) { + upb_status_seterrf(parser->status, "Parse error at '%.*s'\n", pe - p, p); + } else { + capture_suspend(parser, &p); + } + +error: + /* Save parsing state back to parser. */ + parser->current_state = cs; + parser->parser_top = top; + + return p - buf; +} + +static bool end(void *closure, const void *hd) { + upb_json_parser *parser = closure; + + /* Prevent compile warning on unused static constants. */ + UPB_UNUSED(json_start); + UPB_UNUSED(json_en_duration_machine); + UPB_UNUSED(json_en_fieldmask_machine); + UPB_UNUSED(json_en_number_machine); + UPB_UNUSED(json_en_string_machine); + UPB_UNUSED(json_en_timestamp_machine); + UPB_UNUSED(json_en_value_machine); + UPB_UNUSED(json_en_main); + + parse(parser, hd, &eof_ch, 0, NULL); + + return parser->current_state >= %%{ write first_final; }%%; +} + +static void json_parser_reset(upb_json_parser *p) { + int cs; + int top; + + p->top = p->stack; + init_frame(p->top); + + /* Emit Ragel initialization of the parser. */ + %% write init; + p->current_state = cs; + p->parser_top = top; + accumulate_clear(p); + p->multipart_state = MULTIPART_INACTIVE; + p->capture = NULL; + p->accumulated = NULL; +} + +static upb_json_parsermethod *parsermethod_new(upb_json_codecache *c, + const upb_msgdef *md) { + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + upb_json_parsermethod *m = upb_malloc(alloc, sizeof(*m)); + + m->cache = c; + + upb_byteshandler_init(&m->input_handler_); + upb_byteshandler_setstring(&m->input_handler_, parse, m); + upb_byteshandler_setendstr(&m->input_handler_, end, m); + + upb_strtable_init2(&m->name_table, UPB_CTYPE_CONSTPTR, alloc); + + /* Build name_table */ + + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_value v = upb_value_constptr(f); + char *buf; + + /* Add an entry for the JSON name. */ + size_t len = upb_fielddef_getjsonname(f, NULL, 0); + buf = upb_malloc(alloc, len); + upb_fielddef_getjsonname(f, buf, len); + upb_strtable_insert3(&m->name_table, buf, strlen(buf), v, alloc); + + if (strcmp(buf, upb_fielddef_name(f)) != 0) { + /* Since the JSON name is different from the regular field name, add an + * entry for the raw name (compliant proto3 JSON parsers must accept + * both). */ + const char *name = upb_fielddef_name(f); + upb_strtable_insert3(&m->name_table, name, strlen(name), v, alloc); + } + } + + return m; +} + +/* Public API *****************************************************************/ + +upb_json_parser *upb_json_parser_create(upb_arena *arena, + const upb_json_parsermethod *method, + const upb_symtab* symtab, + upb_sink output, + upb_status *status, + bool ignore_json_unknown) { +#ifndef NDEBUG + const size_t size_before = upb_arena_bytesallocated(arena); +#endif + upb_json_parser *p = upb_arena_malloc(arena, sizeof(upb_json_parser)); + if (!p) return false; + + p->arena = arena; + p->method = method; + p->status = status; + p->limit = p->stack + UPB_JSON_MAX_DEPTH; + p->accumulate_buf = NULL; + p->accumulate_buf_size = 0; + upb_bytessink_reset(&p->input_, &method->input_handler_, p); + + json_parser_reset(p); + p->top->sink = output; + p->top->m = upb_handlers_msgdef(output.handlers); + if (is_wellknown_msg(p, UPB_WELLKNOWN_ANY)) { + p->top->is_any = true; + p->top->any_frame = json_parser_any_frame_new(p); + } else { + p->top->is_any = false; + p->top->any_frame = NULL; + } + set_name_table(p, p->top); + p->symtab = symtab; + + p->ignore_json_unknown = ignore_json_unknown; + + /* If this fails, uncomment and increase the value in parser.h. */ + /* fprintf(stderr, "%zd\n", upb_arena_bytesallocated(arena) - size_before); */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= + UPB_JSON_PARSER_SIZE); + return p; +} + +upb_bytessink upb_json_parser_input(upb_json_parser *p) { + return p->input_; +} + +const upb_byteshandler *upb_json_parsermethod_inputhandler( + const upb_json_parsermethod *m) { + return &m->input_handler_; +} + +upb_json_codecache *upb_json_codecache_new(void) { + upb_alloc *alloc; + upb_json_codecache *c; + + c = upb_gmalloc(sizeof(*c)); + + c->arena = upb_arena_new(); + alloc = upb_arena_alloc(c->arena); + + upb_inttable_init2(&c->methods, UPB_CTYPE_CONSTPTR, alloc); + + return c; +} + +void upb_json_codecache_free(upb_json_codecache *c) { + upb_arena_free(c->arena); + upb_gfree(c); +} + +const upb_json_parsermethod *upb_json_codecache_get(upb_json_codecache *c, + const upb_msgdef *md) { + upb_json_parsermethod *m; + upb_value v; + upb_msg_field_iter i; + upb_alloc *alloc = upb_arena_alloc(c->arena); + + if (upb_inttable_lookupptr(&c->methods, md, &v)) { + return upb_value_getconstptr(v); + } + + m = parsermethod_new(c, md); + v = upb_value_constptr(m); + + if (!m) return NULL; + if (!upb_inttable_insertptr2(&c->methods, md, v, alloc)) return NULL; + + /* Populate parser methods for all submessages, so the name tables will + * be available during parsing. */ + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + + if (upb_fielddef_issubmsg(f)) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + const upb_json_parsermethod *sub_method = + upb_json_codecache_get(c, subdef); + + if (!sub_method) return NULL; + } + } + + return m; +} diff --git a/upb/json/printer.c b/upb/json/printer.c new file mode 100644 index 00000000000..38f817d4d40 --- /dev/null +++ b/upb/json/printer.c @@ -0,0 +1,1406 @@ +/* +** This currently uses snprintf() to format primitives, and could be optimized +** further. +*/ + +#include "upb/json/printer.h" + +#include +#include +#include +#include + +#include "upb/port_def.inc" + +struct upb_json_printer { + upb_sink input_; + /* BytesSink closure. */ + void *subc_; + upb_bytessink output_; + + /* We track the depth so that we know when to emit startstr/endstr on the + * output. */ + int depth_; + + /* Have we emitted the first element? This state is necessary to emit commas + * without leaving a trailing comma in arrays/maps. We keep this state per + * frame depth. + * + * Why max_depth * 2? UPB_MAX_HANDLER_DEPTH counts depth as nested messages. + * We count frames (contexts in which we separate elements by commas) as both + * repeated fields and messages (maps), and the worst case is a + * message->repeated field->submessage->repeated field->... nesting. */ + bool first_elem_[UPB_MAX_HANDLER_DEPTH * 2]; + + /* To print timestamp, printer needs to cache its seconds and nanos values + * and convert them when ending timestamp message. See comments of + * printer_sethandlers_timestamp for more detail. */ + int64_t seconds; + int32_t nanos; +}; + +/* StringPiece; a pointer plus a length. */ +typedef struct { + char *ptr; + size_t len; +} strpc; + +void freestrpc(void *ptr) { + strpc *pc = ptr; + upb_gfree(pc->ptr); + upb_gfree(pc); +} + +typedef struct { + bool preserve_fieldnames; +} upb_json_printercache; + +/* Convert fielddef name to JSON name and return as a string piece. */ +strpc *newstrpc(upb_handlers *h, const upb_fielddef *f, + bool preserve_fieldnames) { + /* TODO(haberman): handle malloc failure. */ + strpc *ret = upb_gmalloc(sizeof(*ret)); + if (preserve_fieldnames) { + ret->ptr = upb_gstrdup(upb_fielddef_name(f)); + ret->len = strlen(ret->ptr); + } else { + size_t len; + ret->len = upb_fielddef_getjsonname(f, NULL, 0); + ret->ptr = upb_gmalloc(ret->len); + len = upb_fielddef_getjsonname(f, ret->ptr, ret->len); + UPB_ASSERT(len == ret->len); + ret->len--; /* NULL */ + } + + upb_handlers_addcleanup(h, ret, freestrpc); + return ret; +} + +/* Convert a null-terminated const char* to a string piece. */ +strpc *newstrpc_str(upb_handlers *h, const char * str) { + strpc * ret = upb_gmalloc(sizeof(*ret)); + ret->ptr = upb_gstrdup(str); + ret->len = strlen(str); + upb_handlers_addcleanup(h, ret, freestrpc); + return ret; +} + +/* ------------ JSON string printing: values, maps, arrays ------------------ */ + +static void print_data( + upb_json_printer *p, const char *buf, unsigned int len) { + /* TODO: Will need to change if we support pushback from the sink. */ + size_t n = upb_bytessink_putbuf(p->output_, p->subc_, buf, len, NULL); + UPB_ASSERT(n == len); +} + +static void print_comma(upb_json_printer *p) { + if (!p->first_elem_[p->depth_]) { + print_data(p, ",", 1); + } + p->first_elem_[p->depth_] = false; +} + +/* Helpers that print properly formatted elements to the JSON output stream. */ + +/* Used for escaping control chars in strings. */ +static const char kControlCharLimit = 0x20; + +UPB_INLINE bool is_json_escaped(char c) { + /* See RFC 4627. */ + unsigned char uc = (unsigned char)c; + return uc < kControlCharLimit || uc == '"' || uc == '\\'; +} + +UPB_INLINE const char* json_nice_escape(char c) { + switch (c) { + case '"': return "\\\""; + case '\\': return "\\\\"; + case '\b': return "\\b"; + case '\f': return "\\f"; + case '\n': return "\\n"; + case '\r': return "\\r"; + case '\t': return "\\t"; + default: return NULL; + } +} + +/* Write a properly escaped string chunk. The surrounding quotes are *not* + * printed; this is so that the caller has the option of emitting the string + * content in chunks. */ +static void putstring(upb_json_printer *p, const char *buf, unsigned int len) { + const char* unescaped_run = NULL; + unsigned int i; + for (i = 0; i < len; i++) { + char c = buf[i]; + /* Handle escaping. */ + if (is_json_escaped(c)) { + /* Use a "nice" escape, like \n, if one exists for this character. */ + const char* escape = json_nice_escape(c); + /* If we don't have a specific 'nice' escape code, use a \uXXXX-style + * escape. */ + char escape_buf[8]; + if (!escape) { + unsigned char byte = (unsigned char)c; + _upb_snprintf(escape_buf, sizeof(escape_buf), "\\u%04x", (int)byte); + escape = escape_buf; + } + + /* N.B. that we assume that the input encoding is equal to the output + * encoding (both UTF-8 for now), so for chars >= 0x20 and != \, ", we + * can simply pass the bytes through. */ + + /* If there's a current run of unescaped chars, print that run first. */ + if (unescaped_run) { + print_data(p, unescaped_run, &buf[i] - unescaped_run); + unescaped_run = NULL; + } + /* Then print the escape code. */ + print_data(p, escape, strlen(escape)); + } else { + /* Add to the current unescaped run of characters. */ + if (unescaped_run == NULL) { + unescaped_run = &buf[i]; + } + } + } + + /* If the string ended in a run of unescaped characters, print that last run. */ + if (unescaped_run) { + print_data(p, unescaped_run, &buf[len] - unescaped_run); + } +} + +#define CHKLENGTH(x) if (!(x)) return -1; + +/* Helpers that format floating point values according to our custom formats. + * Right now we use %.8g and %.17g for float/double, respectively, to match + * proto2::util::JsonFormat's defaults. May want to change this later. */ + +const char neginf[] = "\"-Infinity\""; +const char inf[] = "\"Infinity\""; + +static size_t fmt_double(double val, char* buf, size_t length) { + if (val == UPB_INFINITY) { + CHKLENGTH(length >= strlen(inf)); + strcpy(buf, inf); + return strlen(inf); + } else if (val == -UPB_INFINITY) { + CHKLENGTH(length >= strlen(neginf)); + strcpy(buf, neginf); + return strlen(neginf); + } else { + size_t n = _upb_snprintf(buf, length, "%.17g", val); + CHKLENGTH(n > 0 && n < length); + return n; + } +} + +static size_t fmt_float(float val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%.8g", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_bool(bool val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%s", (val ? "true" : "false")); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_int64_as_number(long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%lld", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_uint64_as_number( + unsigned long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "%llu", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_int64_as_string(long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "\"%lld\"", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +static size_t fmt_uint64_as_string( + unsigned long long val, char* buf, size_t length) { + size_t n = _upb_snprintf(buf, length, "\"%llu\"", val); + CHKLENGTH(n > 0 && n < length); + return n; +} + +/* Print a map key given a field name. Called by scalar field handlers and by + * startseq for repeated fields. */ +static bool putkey(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + const strpc *key = handler_data; + print_comma(p); + print_data(p, "\"", 1); + putstring(p, key->ptr, key->len); + print_data(p, "\":", 2); + return true; +} + +#define CHKFMT(val) if ((val) == (size_t)-1) return false; +#define CHK(val) if (!(val)) return false; + +#define TYPE_HANDLERS(type, fmt_func) \ + static bool put##type(void *closure, const void *handler_data, type val) { \ + upb_json_printer *p = closure; \ + char data[64]; \ + size_t length = fmt_func(val, data, sizeof(data)); \ + UPB_UNUSED(handler_data); \ + CHKFMT(length); \ + print_data(p, data, length); \ + return true; \ + } \ + static bool scalar_##type(void *closure, const void *handler_data, \ + type val) { \ + CHK(putkey(closure, handler_data)); \ + CHK(put##type(closure, handler_data, val)); \ + return true; \ + } \ + static bool repeated_##type(void *closure, const void *handler_data, \ + type val) { \ + upb_json_printer *p = closure; \ + print_comma(p); \ + CHK(put##type(closure, handler_data, val)); \ + return true; \ + } + +#define TYPE_HANDLERS_MAPKEY(type, fmt_func) \ + static bool putmapkey_##type(void *closure, const void *handler_data, \ + type val) { \ + upb_json_printer *p = closure; \ + char data[64]; \ + size_t length = fmt_func(val, data, sizeof(data)); \ + UPB_UNUSED(handler_data); \ + print_data(p, "\"", 1); \ + print_data(p, data, length); \ + print_data(p, "\":", 2); \ + return true; \ + } + +TYPE_HANDLERS(double, fmt_double) +TYPE_HANDLERS(float, fmt_float) +TYPE_HANDLERS(bool, fmt_bool) +TYPE_HANDLERS(int32_t, fmt_int64_as_number) +TYPE_HANDLERS(uint32_t, fmt_int64_as_number) +TYPE_HANDLERS(int64_t, fmt_int64_as_string) +TYPE_HANDLERS(uint64_t, fmt_uint64_as_string) + +/* double and float are not allowed to be map keys. */ +TYPE_HANDLERS_MAPKEY(bool, fmt_bool) +TYPE_HANDLERS_MAPKEY(int32_t, fmt_int64_as_number) +TYPE_HANDLERS_MAPKEY(uint32_t, fmt_int64_as_number) +TYPE_HANDLERS_MAPKEY(int64_t, fmt_int64_as_number) +TYPE_HANDLERS_MAPKEY(uint64_t, fmt_uint64_as_number) + +#undef TYPE_HANDLERS +#undef TYPE_HANDLERS_MAPKEY + +typedef struct { + void *keyname; + const upb_enumdef *enumdef; +} EnumHandlerData; + +static bool scalar_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + const char *symbolic_name; + + CHK(putkey(closure, hd->keyname)); + + symbolic_name = upb_enumdef_iton(hd->enumdef, val); + if (symbolic_name) { + print_data(p, "\"", 1); + putstring(p, symbolic_name, strlen(symbolic_name)); + print_data(p, "\"", 1); + } else { + putint32_t(closure, NULL, val); + } + + return true; +} + +static void print_enum_symbolic_name(upb_json_printer *p, + const upb_enumdef *def, + int32_t val) { + const char *symbolic_name = upb_enumdef_iton(def, val); + if (symbolic_name) { + print_data(p, "\"", 1); + putstring(p, symbolic_name, strlen(symbolic_name)); + print_data(p, "\"", 1); + } else { + putint32_t(p, NULL, val); + } +} + +static bool repeated_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + print_comma(p); + + print_enum_symbolic_name(p, hd->enumdef, val); + + return true; +} + +static bool mapvalue_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + + print_enum_symbolic_name(p, hd->enumdef, val); + + return true; +} + +static void *scalar_startsubmsg(void *closure, const void *handler_data) { + return putkey(closure, handler_data) ? closure : UPB_BREAK; +} + +static void *repeated_startsubmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_comma(p); + return closure; +} + +static void start_frame(upb_json_printer *p) { + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); +} + +static void end_frame(upb_json_printer *p) { + print_data(p, "}", 1); + p->depth_--; +} + +static bool printer_startmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + start_frame(p); + return true; +} + +static bool printer_endmsg(void *closure, const void *handler_data, upb_status *s) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + end_frame(p); + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +static void *startseq(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + CHK(putkey(closure, handler_data)); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "[", 1); + return closure; +} + +static bool endseq(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "]", 1); + p->depth_--; + return true; +} + +static void *startmap(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + CHK(putkey(closure, handler_data)); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); + return closure; +} + +static bool endmap(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "}", 1); + p->depth_--; + return true; +} + +static size_t putstr(void *closure, const void *handler_data, const char *str, + size_t len, const upb_bufhandle *handle) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(handle); + putstring(p, str, len); + return len; +} + +/* This has to Base64 encode the bytes, because JSON has no "bytes" type. */ +static size_t putbytes(void *closure, const void *handler_data, const char *str, + size_t len, const upb_bufhandle *handle) { + upb_json_printer *p = closure; + + /* This is the regular base64, not the "web-safe" version. */ + static const char base64[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /* Base64-encode. */ + char data[16000]; + const char *limit = data + sizeof(data); + const unsigned char *from = (const unsigned char*)str; + char *to = data; + size_t remaining = len; + size_t bytes; + + UPB_UNUSED(handler_data); + UPB_UNUSED(handle); + + print_data(p, "\"", 1); + + while (remaining > 2) { + if (limit - to < 4) { + bytes = to - data; + putstring(p, data, bytes); + to = data; + } + + to[0] = base64[from[0] >> 2]; + to[1] = base64[((from[0] & 0x3) << 4) | (from[1] >> 4)]; + to[2] = base64[((from[1] & 0xf) << 2) | (from[2] >> 6)]; + to[3] = base64[from[2] & 0x3f]; + + remaining -= 3; + to += 4; + from += 3; + } + + switch (remaining) { + case 2: + to[0] = base64[from[0] >> 2]; + to[1] = base64[((from[0] & 0x3) << 4) | (from[1] >> 4)]; + to[2] = base64[(from[1] & 0xf) << 2]; + to[3] = '='; + to += 4; + from += 2; + break; + case 1: + to[0] = base64[from[0] >> 2]; + to[1] = base64[((from[0] & 0x3) << 4)]; + to[2] = '='; + to[3] = '='; + to += 4; + from += 1; + break; + } + + bytes = to - data; + putstring(p, data, bytes); + print_data(p, "\"", 1); + return len; +} + +static void *scalar_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + CHK(putkey(closure, handler_data)); + print_data(p, "\"", 1); + return p; +} + +static size_t scalar_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool scalar_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\"", 1); + return true; +} + +static void *repeated_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_comma(p); + print_data(p, "\"", 1); + return p; +} + +static size_t repeated_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool repeated_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\"", 1); + return true; +} + +static void *mapkeyval_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_data(p, "\"", 1); + return p; +} + +static size_t mapkey_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool mapkey_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\":", 2); + return true; +} + +static bool mapvalue_endstr(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + print_data(p, "\"", 1); + return true; +} + +static size_t scalar_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putkey(closure, handler_data)); + CHK(putbytes(closure, handler_data, str, len, handle)); + return len; +} + +static size_t repeated_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + print_comma(p); + CHK(putbytes(closure, handler_data, str, len, handle)); + return len; +} + +static size_t mapkey_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + CHK(putbytes(closure, handler_data, str, len, handle)); + print_data(p, ":", 1); + return len; +} + +static void set_enum_hd(upb_handlers *h, + const upb_fielddef *f, + bool preserve_fieldnames, + upb_handlerattr *attr) { + EnumHandlerData *hd = upb_gmalloc(sizeof(EnumHandlerData)); + hd->enumdef = upb_fielddef_enumsubdef(f); + hd->keyname = newstrpc(h, f, preserve_fieldnames); + upb_handlers_addcleanup(h, hd, upb_gfree); + attr->handler_data = hd; +} + +/* Set up handlers for a mapentry submessage (i.e., an individual key/value pair + * in a map). + * + * TODO: Handle missing key, missing value, out-of-order key/value, or repeated + * key or value cases properly. The right way to do this is to allocate a + * temporary structure at the start of a mapentry submessage, store key and + * value data in it as key and value handlers are called, and then print the + * key/value pair once at the end of the submessage. If we don't do this, we + * should at least detect the case and throw an error. However, so far all of + * our sources that emit mapentry messages do so canonically (with one key + * field, and then one value field), so this is not a pressing concern at the + * moment. */ +void printer_sethandlers_mapentry(const void *closure, bool preserve_fieldnames, + upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + /* A mapentry message is printed simply as '"key": value'. Rather than + * special-case key and value for every type below, we just handle both + * fields explicitly here. */ + const upb_fielddef* key_field = upb_msgdef_itof(md, UPB_MAPENTRY_KEY); + const upb_fielddef* value_field = upb_msgdef_itof(md, UPB_MAPENTRY_VALUE); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + UPB_UNUSED(closure); + + switch (upb_fielddef_type(key_field)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, key_field, putmapkey_int32_t, &empty_attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, key_field, putmapkey_int64_t, &empty_attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, key_field, putmapkey_uint32_t, &empty_attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, key_field, putmapkey_uint64_t, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, key_field, putmapkey_bool, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, key_field, mapkeyval_startstr, &empty_attr); + upb_handlers_setstring(h, key_field, mapkey_str, &empty_attr); + upb_handlers_setendstr(h, key_field, mapkey_endstr, &empty_attr); + break; + case UPB_TYPE_BYTES: + upb_handlers_setstring(h, key_field, mapkey_bytes, &empty_attr); + break; + default: + UPB_ASSERT(false); + break; + } + + switch (upb_fielddef_type(value_field)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, value_field, putint32_t, &empty_attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, value_field, putint64_t, &empty_attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, value_field, putuint32_t, &empty_attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, value_field, putuint64_t, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, value_field, putbool, &empty_attr); + break; + case UPB_TYPE_FLOAT: + upb_handlers_setfloat(h, value_field, putfloat, &empty_attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, value_field, putdouble, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, value_field, mapkeyval_startstr, &empty_attr); + upb_handlers_setstring(h, value_field, putstr, &empty_attr); + upb_handlers_setendstr(h, value_field, mapvalue_endstr, &empty_attr); + break; + case UPB_TYPE_BYTES: + upb_handlers_setstring(h, value_field, putbytes, &empty_attr); + break; + case UPB_TYPE_ENUM: { + upb_handlerattr enum_attr = UPB_HANDLERATTR_INIT; + set_enum_hd(h, value_field, preserve_fieldnames, &enum_attr); + upb_handlers_setint32(h, value_field, mapvalue_enum, &enum_attr); + break; + } + case UPB_TYPE_MESSAGE: + /* No handler necessary -- the submsg handlers will print the message + * as appropriate. */ + break; + } +} + +static bool putseconds(void *closure, const void *handler_data, + int64_t seconds) { + upb_json_printer *p = closure; + p->seconds = seconds; + UPB_UNUSED(handler_data); + return true; +} + +static bool putnanos(void *closure, const void *handler_data, + int32_t nanos) { + upb_json_printer *p = closure; + p->nanos = nanos; + UPB_UNUSED(handler_data); + return true; +} + +static void *scalar_startstr_nokey(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_data(p, "\"", 1); + return p; +} + +static size_t putstr_nokey(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(handle); + print_data(p, "\"", 1); + putstring(p, str, len); + print_data(p, "\"", 1); + return len + 2; +} + +static void *startseq_nokey(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "[", 1); + return closure; +} + +static void *startseq_fieldmask(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_++; + p->first_elem_[p->depth_] = true; + return closure; +} + +static bool endseq_fieldmask(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_--; + return true; +} + +static void *repeated_startstr_fieldmask( + void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + print_comma(p); + return p; +} + +static size_t repeated_str_fieldmask( + void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + const char* limit = str + len; + bool upper = false; + size_t result_len = 0; + for (; str < limit; str++) { + if (*str == '_') { + upper = true; + continue; + } + if (upper && *str >= 'a' && *str <= 'z') { + char upper_char = toupper(*str); + CHK(putstr(closure, handler_data, &upper_char, 1, handle)); + } else { + CHK(putstr(closure, handler_data, str, 1, handle)); + } + upper = false; + result_len++; + } + return result_len; +} + +static void *startmap_nokey(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); + return closure; +} + +static bool putnull(void *closure, const void *handler_data, + int32_t null) { + upb_json_printer *p = closure; + print_data(p, "null", 4); + UPB_UNUSED(handler_data); + UPB_UNUSED(null); + return true; +} + +static bool printer_startdurationmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + return true; +} + +#define UPB_DURATION_MAX_JSON_LEN 23 +#define UPB_DURATION_MAX_NANO_LEN 9 + +static bool printer_enddurationmsg(void *closure, const void *handler_data, + upb_status *s) { + upb_json_printer *p = closure; + char buffer[UPB_DURATION_MAX_JSON_LEN]; + size_t base_len; + size_t curr; + size_t i; + + memset(buffer, 0, UPB_DURATION_MAX_JSON_LEN); + + if (p->seconds < -315576000000) { + upb_status_seterrf(s, "error parsing duration: " + "minimum acceptable value is " + "-315576000000"); + return false; + } + + if (p->seconds > 315576000000) { + upb_status_seterrf(s, "error serializing duration: " + "maximum acceptable value is " + "315576000000"); + return false; + } + + _upb_snprintf(buffer, sizeof(buffer), "%ld", (long)p->seconds); + base_len = strlen(buffer); + + if (p->nanos != 0) { + char nanos_buffer[UPB_DURATION_MAX_NANO_LEN + 3]; + _upb_snprintf(nanos_buffer, sizeof(nanos_buffer), "%.9f", + p->nanos / 1000000000.0); + /* Remove trailing 0. */ + for (i = UPB_DURATION_MAX_NANO_LEN + 2; + nanos_buffer[i] == '0'; i--) { + nanos_buffer[i] = 0; + } + strcpy(buffer + base_len, nanos_buffer + 1); + } + + curr = strlen(buffer); + strcpy(buffer + curr, "s"); + + p->seconds = 0; + p->nanos = 0; + + print_data(p, "\"", 1); + print_data(p, buffer, strlen(buffer)); + print_data(p, "\"", 1); + + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + + UPB_UNUSED(handler_data); + return true; +} + +static bool printer_starttimestampmsg(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + return true; +} + +#define UPB_TIMESTAMP_MAX_JSON_LEN 31 +#define UPB_TIMESTAMP_BEFORE_NANO_LEN 19 +#define UPB_TIMESTAMP_MAX_NANO_LEN 9 + +static bool printer_endtimestampmsg(void *closure, const void *handler_data, + upb_status *s) { + upb_json_printer *p = closure; + char buffer[UPB_TIMESTAMP_MAX_JSON_LEN]; + time_t time = p->seconds; + size_t curr; + size_t i; + size_t year_length = + strftime(buffer, UPB_TIMESTAMP_MAX_JSON_LEN, "%Y", gmtime(&time)); + + if (p->seconds < -62135596800) { + upb_status_seterrf(s, "error parsing timestamp: " + "minimum acceptable value is " + "0001-01-01T00:00:00Z"); + return false; + } + + if (p->seconds > 253402300799) { + upb_status_seterrf(s, "error parsing timestamp: " + "maximum acceptable value is " + "9999-12-31T23:59:59Z"); + return false; + } + + /* strftime doesn't guarantee 4 digits for year. Prepend 0 by ourselves. */ + for (i = 0; i < 4 - year_length; i++) { + buffer[i] = '0'; + } + + strftime(buffer + (4 - year_length), UPB_TIMESTAMP_MAX_JSON_LEN, + "%Y-%m-%dT%H:%M:%S", gmtime(&time)); + if (p->nanos != 0) { + char nanos_buffer[UPB_TIMESTAMP_MAX_NANO_LEN + 3]; + _upb_snprintf(nanos_buffer, sizeof(nanos_buffer), "%.9f", + p->nanos / 1000000000.0); + /* Remove trailing 0. */ + for (i = UPB_TIMESTAMP_MAX_NANO_LEN + 2; + nanos_buffer[i] == '0'; i--) { + nanos_buffer[i] = 0; + } + strcpy(buffer + UPB_TIMESTAMP_BEFORE_NANO_LEN, nanos_buffer + 1); + } + + curr = strlen(buffer); + strcpy(buffer + curr, "Z"); + + p->seconds = 0; + p->nanos = 0; + + print_data(p, "\"", 1); + print_data(p, buffer, strlen(buffer)); + print_data(p, "\"", 1); + + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + return true; +} + +static bool printer_startmsg_noframe(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + return true; +} + +static bool printer_endmsg_noframe( + void *closure, const void *handler_data, upb_status *s) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +static bool printer_startmsg_fieldmask( + void *closure, const void *handler_data) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + if (p->depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc_); + } + print_data(p, "\"", 1); + return true; +} + +static bool printer_endmsg_fieldmask( + void *closure, const void *handler_data, upb_status *s) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(s); + print_data(p, "\"", 1); + if (p->depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +static void *scalar_startstr_onlykey( + void *closure, const void *handler_data, size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(size_hint); + CHK(putkey(closure, handler_data)); + return p; +} + +/* Set up handlers for an Any submessage. */ +void printer_sethandlers_any(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* type_field = upb_msgdef_itof(md, UPB_ANY_TYPE); + const upb_fielddef* value_field = upb_msgdef_itof(md, UPB_ANY_VALUE); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + /* type_url's json name is "@type" */ + upb_handlerattr type_name_attr = UPB_HANDLERATTR_INIT; + upb_handlerattr value_name_attr = UPB_HANDLERATTR_INIT; + strpc *type_url_json_name = newstrpc_str(h, "@type"); + strpc *value_json_name = newstrpc_str(h, "value"); + + type_name_attr.handler_data = type_url_json_name; + value_name_attr.handler_data = value_json_name; + + /* Set up handlers. */ + upb_handlers_setstartmsg(h, printer_startmsg, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg, &empty_attr); + + upb_handlers_setstartstr(h, type_field, scalar_startstr, &type_name_attr); + upb_handlers_setstring(h, type_field, scalar_str, &empty_attr); + upb_handlers_setendstr(h, type_field, scalar_endstr, &empty_attr); + + /* This is not the full and correct JSON encoding for the Any value field. It + * requires further processing by the wrapper code based on the type URL. + */ + upb_handlers_setstartstr(h, value_field, scalar_startstr_onlykey, + &value_name_attr); + + UPB_UNUSED(closure); +} + +/* Set up handlers for a fieldmask submessage. */ +void printer_sethandlers_fieldmask(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + const upb_fielddef* f = upb_msgdef_itof(md, 1); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartseq(h, f, startseq_fieldmask, &empty_attr); + upb_handlers_setendseq(h, f, endseq_fieldmask, &empty_attr); + + upb_handlers_setstartmsg(h, printer_startmsg_fieldmask, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_fieldmask, &empty_attr); + + upb_handlers_setstartstr(h, f, repeated_startstr_fieldmask, &empty_attr); + upb_handlers_setstring(h, f, repeated_str_fieldmask, &empty_attr); + + UPB_UNUSED(closure); +} + +/* Set up handlers for a duration submessage. */ +void printer_sethandlers_duration(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* seconds_field = + upb_msgdef_itof(md, UPB_DURATION_SECONDS); + const upb_fielddef* nanos_field = + upb_msgdef_itof(md, UPB_DURATION_NANOS); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartmsg(h, printer_startdurationmsg, &empty_attr); + upb_handlers_setint64(h, seconds_field, putseconds, &empty_attr); + upb_handlers_setint32(h, nanos_field, putnanos, &empty_attr); + upb_handlers_setendmsg(h, printer_enddurationmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +/* Set up handlers for a timestamp submessage. Instead of printing fields + * separately, the json representation of timestamp follows RFC 3339 */ +void printer_sethandlers_timestamp(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* seconds_field = + upb_msgdef_itof(md, UPB_TIMESTAMP_SECONDS); + const upb_fielddef* nanos_field = + upb_msgdef_itof(md, UPB_TIMESTAMP_NANOS); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartmsg(h, printer_starttimestampmsg, &empty_attr); + upb_handlers_setint64(h, seconds_field, putseconds, &empty_attr); + upb_handlers_setint32(h, nanos_field, putnanos, &empty_attr); + upb_handlers_setendmsg(h, printer_endtimestampmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +void printer_sethandlers_value(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + upb_msg_field_iter i; + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); + + upb_msg_field_begin(&i, md); + for(; !upb_msg_field_done(&i); upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_ENUM: + upb_handlers_setint32(h, f, putnull, &empty_attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, f, putdouble, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, f, scalar_startstr_nokey, &empty_attr); + upb_handlers_setstring(h, f, scalar_str, &empty_attr); + upb_handlers_setendstr(h, f, scalar_endstr, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, f, putbool, &empty_attr); + break; + case UPB_TYPE_MESSAGE: + break; + default: + UPB_ASSERT(false); + break; + } + } + + UPB_UNUSED(closure); +} + +#define WRAPPER_SETHANDLERS(wrapper, type, putmethod) \ +void printer_sethandlers_##wrapper(const void *closure, upb_handlers *h) { \ + const upb_msgdef *md = upb_handlers_msgdef(h); \ + const upb_fielddef* f = upb_msgdef_itof(md, 1); \ + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; \ + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); \ + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); \ + upb_handlers_set##type(h, f, putmethod, &empty_attr); \ + UPB_UNUSED(closure); \ +} + +WRAPPER_SETHANDLERS(doublevalue, double, putdouble) +WRAPPER_SETHANDLERS(floatvalue, float, putfloat) +WRAPPER_SETHANDLERS(int64value, int64, putint64_t) +WRAPPER_SETHANDLERS(uint64value, uint64, putuint64_t) +WRAPPER_SETHANDLERS(int32value, int32, putint32_t) +WRAPPER_SETHANDLERS(uint32value, uint32, putuint32_t) +WRAPPER_SETHANDLERS(boolvalue, bool, putbool) +WRAPPER_SETHANDLERS(stringvalue, string, putstr_nokey) +WRAPPER_SETHANDLERS(bytesvalue, string, putbytes) + +#undef WRAPPER_SETHANDLERS + +void printer_sethandlers_listvalue(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + const upb_fielddef* f = upb_msgdef_itof(md, 1); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartseq(h, f, startseq_nokey, &empty_attr); + upb_handlers_setendseq(h, f, endseq, &empty_attr); + + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); + + upb_handlers_setstartsubmsg(h, f, repeated_startsubmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +void printer_sethandlers_structvalue(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + const upb_fielddef* f = upb_msgdef_itof(md, 1); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + + upb_handlers_setstartseq(h, f, startmap_nokey, &empty_attr); + upb_handlers_setendseq(h, f, endmap, &empty_attr); + + upb_handlers_setstartmsg(h, printer_startmsg_noframe, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg_noframe, &empty_attr); + + upb_handlers_setstartsubmsg(h, f, repeated_startsubmsg, &empty_attr); + + UPB_UNUSED(closure); +} + +void printer_sethandlers(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + bool is_mapentry = upb_msgdef_mapentry(md); + upb_handlerattr empty_attr = UPB_HANDLERATTR_INIT; + upb_msg_field_iter i; + const upb_json_printercache *cache = closure; + const bool preserve_fieldnames = cache->preserve_fieldnames; + + if (is_mapentry) { + /* mapentry messages are sufficiently different that we handle them + * separately. */ + printer_sethandlers_mapentry(closure, preserve_fieldnames, h); + return; + } + + switch (upb_msgdef_wellknowntype(md)) { + case UPB_WELLKNOWN_UNSPECIFIED: + break; + case UPB_WELLKNOWN_ANY: + printer_sethandlers_any(closure, h); + return; + case UPB_WELLKNOWN_FIELDMASK: + printer_sethandlers_fieldmask(closure, h); + return; + case UPB_WELLKNOWN_DURATION: + printer_sethandlers_duration(closure, h); + return; + case UPB_WELLKNOWN_TIMESTAMP: + printer_sethandlers_timestamp(closure, h); + return; + case UPB_WELLKNOWN_VALUE: + printer_sethandlers_value(closure, h); + return; + case UPB_WELLKNOWN_LISTVALUE: + printer_sethandlers_listvalue(closure, h); + return; + case UPB_WELLKNOWN_STRUCT: + printer_sethandlers_structvalue(closure, h); + return; +#define WRAPPER(wellknowntype, name) \ + case wellknowntype: \ + printer_sethandlers_##name(closure, h); \ + return; \ + + WRAPPER(UPB_WELLKNOWN_DOUBLEVALUE, doublevalue); + WRAPPER(UPB_WELLKNOWN_FLOATVALUE, floatvalue); + WRAPPER(UPB_WELLKNOWN_INT64VALUE, int64value); + WRAPPER(UPB_WELLKNOWN_UINT64VALUE, uint64value); + WRAPPER(UPB_WELLKNOWN_INT32VALUE, int32value); + WRAPPER(UPB_WELLKNOWN_UINT32VALUE, uint32value); + WRAPPER(UPB_WELLKNOWN_BOOLVALUE, boolvalue); + WRAPPER(UPB_WELLKNOWN_STRINGVALUE, stringvalue); + WRAPPER(UPB_WELLKNOWN_BYTESVALUE, bytesvalue); + +#undef WRAPPER + } + + upb_handlers_setstartmsg(h, printer_startmsg, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg, &empty_attr); + +#define TYPE(type, name, ctype) \ + case type: \ + if (upb_fielddef_isseq(f)) { \ + upb_handlers_set##name(h, f, repeated_##ctype, &empty_attr); \ + } else { \ + upb_handlers_set##name(h, f, scalar_##ctype, &name_attr); \ + } \ + break; + + upb_msg_field_begin(&i, md); + for(; !upb_msg_field_done(&i); upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + + upb_handlerattr name_attr = UPB_HANDLERATTR_INIT; + name_attr.handler_data = newstrpc(h, f, preserve_fieldnames); + + if (upb_fielddef_ismap(f)) { + upb_handlers_setstartseq(h, f, startmap, &name_attr); + upb_handlers_setendseq(h, f, endmap, &name_attr); + } else if (upb_fielddef_isseq(f)) { + upb_handlers_setstartseq(h, f, startseq, &name_attr); + upb_handlers_setendseq(h, f, endseq, &empty_attr); + } + + switch (upb_fielddef_type(f)) { + TYPE(UPB_TYPE_FLOAT, float, float); + TYPE(UPB_TYPE_DOUBLE, double, double); + TYPE(UPB_TYPE_BOOL, bool, bool); + TYPE(UPB_TYPE_INT32, int32, int32_t); + TYPE(UPB_TYPE_UINT32, uint32, uint32_t); + TYPE(UPB_TYPE_INT64, int64, int64_t); + TYPE(UPB_TYPE_UINT64, uint64, uint64_t); + case UPB_TYPE_ENUM: { + /* For now, we always emit symbolic names for enums. We may want an + * option later to control this behavior, but we will wait for a real + * need first. */ + upb_handlerattr enum_attr = UPB_HANDLERATTR_INIT; + set_enum_hd(h, f, preserve_fieldnames, &enum_attr); + + if (upb_fielddef_isseq(f)) { + upb_handlers_setint32(h, f, repeated_enum, &enum_attr); + } else { + upb_handlers_setint32(h, f, scalar_enum, &enum_attr); + } + + break; + } + case UPB_TYPE_STRING: + if (upb_fielddef_isseq(f)) { + upb_handlers_setstartstr(h, f, repeated_startstr, &empty_attr); + upb_handlers_setstring(h, f, repeated_str, &empty_attr); + upb_handlers_setendstr(h, f, repeated_endstr, &empty_attr); + } else { + upb_handlers_setstartstr(h, f, scalar_startstr, &name_attr); + upb_handlers_setstring(h, f, scalar_str, &empty_attr); + upb_handlers_setendstr(h, f, scalar_endstr, &empty_attr); + } + break; + case UPB_TYPE_BYTES: + /* XXX: this doesn't support strings that span buffers yet. The base64 + * encoder will need to be made resumable for this to work properly. */ + if (upb_fielddef_isseq(f)) { + upb_handlers_setstring(h, f, repeated_bytes, &empty_attr); + } else { + upb_handlers_setstring(h, f, scalar_bytes, &name_attr); + } + break; + case UPB_TYPE_MESSAGE: + if (upb_fielddef_isseq(f)) { + upb_handlers_setstartsubmsg(h, f, repeated_startsubmsg, &name_attr); + } else { + upb_handlers_setstartsubmsg(h, f, scalar_startsubmsg, &name_attr); + } + break; + } + } + +#undef TYPE +} + +static void json_printer_reset(upb_json_printer *p) { + p->depth_ = 0; +} + + +/* Public API *****************************************************************/ + +upb_json_printer *upb_json_printer_create(upb_arena *a, const upb_handlers *h, + upb_bytessink output) { +#ifndef NDEBUG + size_t size_before = upb_arena_bytesallocated(a); +#endif + + upb_json_printer *p = upb_arena_malloc(a, sizeof(upb_json_printer)); + if (!p) return NULL; + + p->output_ = output; + json_printer_reset(p); + upb_sink_reset(&p->input_, h, p); + p->seconds = 0; + p->nanos = 0; + + /* If this fails, increase the value in printer.h. */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(a) - size_before <= + UPB_JSON_PRINTER_SIZE); + return p; +} + +upb_sink upb_json_printer_input(upb_json_printer *p) { + return p->input_; +} + +upb_handlercache *upb_json_printer_newcache(bool preserve_proto_fieldnames) { + upb_json_printercache *cache = upb_gmalloc(sizeof(*cache)); + upb_handlercache *ret = upb_handlercache_new(printer_sethandlers, cache); + + cache->preserve_fieldnames = preserve_proto_fieldnames; + upb_handlercache_addcleanup(ret, cache, upb_gfree); + + return ret; +} diff --git a/upb/json/printer.h b/upb/json/printer.h new file mode 100644 index 00000000000..85b9b128f97 --- /dev/null +++ b/upb/json/printer.h @@ -0,0 +1,72 @@ +/* +** upb::json::Printer +** +** Handlers that emit JSON according to a specific protobuf schema. +*/ + +#ifndef UPB_JSON_TYPED_PRINTER_H_ +#define UPB_JSON_TYPED_PRINTER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace json { +class PrinterPtr; +} /* namespace json */ +} /* namespace upb */ +#endif + +/* upb_json_printer ***********************************************************/ + +#define UPB_JSON_PRINTER_SIZE 192 + +struct upb_json_printer; +typedef struct upb_json_printer upb_json_printer; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Native C API. */ +upb_json_printer *upb_json_printer_create(upb_arena *a, const upb_handlers *h, + upb_bytessink output); +upb_sink upb_json_printer_input(upb_json_printer *p); +const upb_handlers *upb_json_printer_newhandlers(const upb_msgdef *md, + bool preserve_fieldnames, + const void *owner); + +/* Lazily builds and caches handlers that will push encoded data to a bytessink. + * Any msgdef objects used with this object must outlive it. */ +upb_handlercache *upb_json_printer_newcache(bool preserve_proto_fieldnames); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Prints an incoming stream of data to a BytesSink in JSON format. */ +class upb::json::PrinterPtr { + public: + PrinterPtr(upb_json_printer* ptr) : ptr_(ptr) {} + + static PrinterPtr Create(Arena *arena, const upb::Handlers *handlers, + BytesSink output) { + return PrinterPtr( + upb_json_printer_create(arena->ptr(), handlers, output.sink())); + } + + /* The input to the printer. */ + Sink input() { return upb_json_printer_input(ptr_); } + + static const size_t kSize = UPB_JSON_PRINTER_SIZE; + + static HandlerCache NewCache(bool preserve_proto_fieldnames) { + return upb_json_printer_newcache(preserve_proto_fieldnames); + } + + private: + upb_json_printer* ptr_; +}; + +#endif /* __cplusplus */ + +#endif /* UPB_JSON_TYPED_PRINTER_H_ */ diff --git a/upb/legacy_msg_reflection.c b/upb/legacy_msg_reflection.c new file mode 100644 index 00000000000..031aa4e9cc7 --- /dev/null +++ b/upb/legacy_msg_reflection.c @@ -0,0 +1,399 @@ + +#include "upb/legacy_msg_reflection.h" + +#include +#include "upb/table.int.h" +#include "upb/msg.h" + +#include "upb/port_def.inc" + +bool upb_fieldtype_mapkeyok(upb_fieldtype_t type) { + return type == UPB_TYPE_BOOL || type == UPB_TYPE_INT32 || + type == UPB_TYPE_UINT32 || type == UPB_TYPE_INT64 || + type == UPB_TYPE_UINT64 || type == UPB_TYPE_STRING; +} + +#define PTR_AT(msg, ofs, type) (type*)((char*)msg + ofs) +#define VOIDPTR_AT(msg, ofs) PTR_AT(msg, ofs, void) +#define ENCODE_MAX_NESTING 64 +#define CHECK_TRUE(x) if (!(x)) { return false; } + +/** upb_msgval ****************************************************************/ + +/* These functions will generate real memcpy() calls on ARM sadly, because + * the compiler assumes they might not be aligned. */ + +static upb_msgval upb_msgval_read(const void *p, size_t ofs, + uint8_t size) { + upb_msgval val; + p = (char*)p + ofs; + memcpy(&val, p, size); + return val; +} + +static void upb_msgval_write(void *p, size_t ofs, upb_msgval val, + uint8_t size) { + p = (char*)p + ofs; + memcpy(p, &val, size); +} + +static size_t upb_msgval_sizeof(upb_fieldtype_t type) { + switch (type) { + case UPB_TYPE_DOUBLE: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + return 8; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_FLOAT: + return 4; + case UPB_TYPE_BOOL: + return 1; + case UPB_TYPE_MESSAGE: + return sizeof(void*); + case UPB_TYPE_BYTES: + case UPB_TYPE_STRING: + return sizeof(upb_strview); + } + UPB_UNREACHABLE(); +} + +static uint8_t upb_msg_fieldsize(const upb_msglayout_field *field) { + if (field->label == UPB_LABEL_REPEATED) { + return sizeof(void*); + } else { + return upb_msgval_sizeof(upb_desctype_to_fieldtype[field->descriptortype]); + } +} + +/* TODO(haberman): this is broken right now because upb_msgval can contain + * a char* / size_t pair, which is too big for a upb_value. To fix this + * we'll probably need to dynamically allocate a upb_msgval and store a + * pointer to that in the tables for extensions/maps. */ +static upb_value upb_toval(upb_msgval val) { + upb_value ret; + UPB_UNUSED(val); + memset(&ret, 0, sizeof(upb_value)); /* XXX */ + return ret; +} + +static upb_msgval upb_msgval_fromval(upb_value val) { + upb_msgval ret; + UPB_UNUSED(val); + memset(&ret, 0, sizeof(upb_msgval)); /* XXX */ + return ret; +} + +static upb_ctype_t upb_fieldtotabtype(upb_fieldtype_t type) { + switch (type) { + case UPB_TYPE_FLOAT: return UPB_CTYPE_FLOAT; + case UPB_TYPE_DOUBLE: return UPB_CTYPE_DOUBLE; + case UPB_TYPE_BOOL: return UPB_CTYPE_BOOL; + case UPB_TYPE_BYTES: + case UPB_TYPE_MESSAGE: + case UPB_TYPE_STRING: return UPB_CTYPE_CONSTPTR; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: return UPB_CTYPE_INT32; + case UPB_TYPE_UINT32: return UPB_CTYPE_UINT32; + case UPB_TYPE_INT64: return UPB_CTYPE_INT64; + case UPB_TYPE_UINT64: return UPB_CTYPE_UINT64; + default: UPB_ASSERT(false); return 0; + } +} + + +/** upb_msg *******************************************************************/ + +/* If we always read/write as a consistent type to each address, this shouldn't + * violate aliasing. + */ +#define DEREF(msg, ofs, type) *PTR_AT(msg, ofs, type) + +static const upb_msglayout_field *upb_msg_checkfield(int field_index, + const upb_msglayout *l) { + UPB_ASSERT(field_index >= 0 && field_index < l->field_count); + return &l->fields[field_index]; +} + +static bool upb_msg_inoneof(const upb_msglayout_field *field) { + return field->presence < 0; +} + +static uint32_t *upb_msg_oneofcase(const upb_msg *msg, int field_index, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + UPB_ASSERT(upb_msg_inoneof(field)); + return PTR_AT(msg, ~field->presence, uint32_t); +} + +bool upb_msg_has(const upb_msg *msg, + int field_index, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + + UPB_ASSERT(field->presence); + + if (upb_msg_inoneof(field)) { + /* Oneofs are set when the oneof number is set to this field. */ + return *upb_msg_oneofcase(msg, field_index, l) == field->number; + } else { + /* Other fields are set when their hasbit is set. */ + uint32_t hasbit = field->presence; + return DEREF(msg, hasbit / 8, char) | (1 << (hasbit % 8)); + } +} + +upb_msgval upb_msg_get(const upb_msg *msg, int field_index, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + int size = upb_msg_fieldsize(field); + return upb_msgval_read(msg, field->offset, size); +} + +void upb_msg_set(upb_msg *msg, int field_index, upb_msgval val, + const upb_msglayout *l) { + const upb_msglayout_field *field = upb_msg_checkfield(field_index, l); + int size = upb_msg_fieldsize(field); + upb_msgval_write(msg, field->offset, val, size); +} + + +/** upb_array *****************************************************************/ + +#define DEREF_ARR(arr, i, type) ((type*)arr->data)[i] + +size_t upb_array_size(const upb_array *arr) { + return arr->len; +} + +upb_msgval upb_array_get(const upb_array *arr, upb_fieldtype_t type, size_t i) { + size_t element_size = upb_msgval_sizeof(type); + UPB_ASSERT(i < arr->len); + return upb_msgval_read(arr->data, i * element_size, element_size); +} + +bool upb_array_set(upb_array *arr, upb_fieldtype_t type, size_t i, + upb_msgval val, upb_arena *arena) { + size_t element_size = upb_msgval_sizeof(type); + UPB_ASSERT(i <= arr->len); + + if (i == arr->len) { + /* Extending the array. */ + + if (i == arr->size) { + /* Need to reallocate. */ + size_t new_size = UPB_MAX(arr->size * 2, 8); + size_t new_bytes = new_size * element_size; + size_t old_bytes = arr->size * element_size; + upb_alloc *alloc = upb_arena_alloc(arena); + upb_msgval *new_data = + upb_realloc(alloc, arr->data, old_bytes, new_bytes); + + if (!new_data) { + return false; + } + + arr->data = new_data; + arr->size = new_size; + } + + arr->len = i + 1; + } + + upb_msgval_write(arr->data, i * element_size, val, element_size); + return true; +} + +/** upb_map *******************************************************************/ + +struct upb_map { + upb_fieldtype_t key_type; + upb_fieldtype_t val_type; + /* We may want to optimize this to use inttable where possible, for greater + * efficiency and lower memory footprint. */ + upb_strtable strtab; + upb_arena *arena; +}; + +static void upb_map_tokey(upb_fieldtype_t type, upb_msgval *key, + const char **out_key, size_t *out_len) { + switch (type) { + case UPB_TYPE_STRING: + /* Point to string data of the input key. */ + *out_key = key->str.data; + *out_len = key->str.size; + return; + case UPB_TYPE_BOOL: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + /* Point to the key itself. XXX: big-endian. */ + *out_key = (const char*)key; + *out_len = upb_msgval_sizeof(type); + return; + case UPB_TYPE_BYTES: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_ENUM: + case UPB_TYPE_FLOAT: + case UPB_TYPE_MESSAGE: + break; /* Cannot be a map key. */ + } + UPB_UNREACHABLE(); +} + +static upb_msgval upb_map_fromkey(upb_fieldtype_t type, const char *key, + size_t len) { + switch (type) { + case UPB_TYPE_STRING: + return upb_msgval_makestr(key, len); + case UPB_TYPE_BOOL: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + return upb_msgval_read(key, 0, upb_msgval_sizeof(type)); + case UPB_TYPE_BYTES: + case UPB_TYPE_DOUBLE: + case UPB_TYPE_ENUM: + case UPB_TYPE_FLOAT: + case UPB_TYPE_MESSAGE: + break; /* Cannot be a map key. */ + } + UPB_UNREACHABLE(); +} + +upb_map *upb_map_new(upb_fieldtype_t ktype, upb_fieldtype_t vtype, + upb_arena *a) { + upb_ctype_t vtabtype = upb_fieldtotabtype(vtype); + upb_alloc *alloc = upb_arena_alloc(a); + upb_map *map = upb_malloc(alloc, sizeof(upb_map)); + + if (!map) { + return NULL; + } + + UPB_ASSERT(upb_fieldtype_mapkeyok(ktype)); + map->key_type = ktype; + map->val_type = vtype; + map->arena = a; + + if (!upb_strtable_init2(&map->strtab, vtabtype, alloc)) { + return NULL; + } + + return map; +} + +size_t upb_map_size(const upb_map *map) { + return upb_strtable_count(&map->strtab); +} + +upb_fieldtype_t upb_map_keytype(const upb_map *map) { + return map->key_type; +} + +upb_fieldtype_t upb_map_valuetype(const upb_map *map) { + return map->val_type; +} + +bool upb_map_get(const upb_map *map, upb_msgval key, upb_msgval *val) { + upb_value tabval; + const char *key_str; + size_t key_len; + bool ret; + + upb_map_tokey(map->key_type, &key, &key_str, &key_len); + ret = upb_strtable_lookup2(&map->strtab, key_str, key_len, &tabval); + if (ret) { + memcpy(val, &tabval, sizeof(tabval)); + } + + return ret; +} + +bool upb_map_set(upb_map *map, upb_msgval key, upb_msgval val, + upb_msgval *removed) { + const char *key_str; + size_t key_len; + upb_value tabval = upb_toval(val); + upb_value removedtabval; + upb_alloc *a = upb_arena_alloc(map->arena); + + upb_map_tokey(map->key_type, &key, &key_str, &key_len); + + /* TODO(haberman): add overwrite operation to minimize number of lookups. */ + if (upb_strtable_lookup2(&map->strtab, key_str, key_len, NULL)) { + upb_strtable_remove3(&map->strtab, key_str, key_len, &removedtabval, a); + memcpy(&removed, &removedtabval, sizeof(removed)); + } + + return upb_strtable_insert3(&map->strtab, key_str, key_len, tabval, a); +} + +bool upb_map_del(upb_map *map, upb_msgval key) { + const char *key_str; + size_t key_len; + upb_alloc *a = upb_arena_alloc(map->arena); + + upb_map_tokey(map->key_type, &key, &key_str, &key_len); + return upb_strtable_remove3(&map->strtab, key_str, key_len, NULL, a); +} + + +/** upb_mapiter ***************************************************************/ + +struct upb_mapiter { + upb_strtable_iter iter; + upb_fieldtype_t key_type; +}; + +size_t upb_mapiter_sizeof(void) { + return sizeof(upb_mapiter); +} + +void upb_mapiter_begin(upb_mapiter *i, const upb_map *map) { + upb_strtable_begin(&i->iter, &map->strtab); + i->key_type = map->key_type; +} + +upb_mapiter *upb_mapiter_new(const upb_map *t, upb_alloc *a) { + upb_mapiter *ret = upb_malloc(a, upb_mapiter_sizeof()); + + if (!ret) { + return NULL; + } + + upb_mapiter_begin(ret, t); + return ret; +} + +void upb_mapiter_free(upb_mapiter *i, upb_alloc *a) { + upb_free(a, i); +} + +void upb_mapiter_next(upb_mapiter *i) { + upb_strtable_next(&i->iter); +} + +bool upb_mapiter_done(const upb_mapiter *i) { + return upb_strtable_done(&i->iter); +} + +upb_msgval upb_mapiter_key(const upb_mapiter *i) { + return upb_map_fromkey(i->key_type, upb_strtable_iter_key(&i->iter), + upb_strtable_iter_keylength(&i->iter)); +} + +upb_msgval upb_mapiter_value(const upb_mapiter *i) { + return upb_msgval_fromval(upb_strtable_iter_value(&i->iter)); +} + +void upb_mapiter_setdone(upb_mapiter *i) { + upb_strtable_iter_setdone(&i->iter); +} + +bool upb_mapiter_isequal(const upb_mapiter *i1, const upb_mapiter *i2) { + return upb_strtable_iter_isequal(&i1->iter, &i2->iter); +} diff --git a/upb/legacy_msg_reflection.h b/upb/legacy_msg_reflection.h new file mode 100644 index 00000000000..c54bfb947a8 --- /dev/null +++ b/upb/legacy_msg_reflection.h @@ -0,0 +1,191 @@ + +#ifndef UPB_LEGACY_MSG_REFLECTION_H_ +#define UPB_LEGACY_MSG_REFLECTION_H_ + +#include "upb/upb.h" +#include "upb/msg.h" + +#include "upb/port_def.inc" + +struct upb_map; +typedef struct upb_map upb_map; + +struct upb_mapiter; +typedef struct upb_mapiter upb_mapiter; + +/** upb_msgval ****************************************************************/ + +/* A union representing all possible protobuf values. Used for generic get/set + * operations. */ + +typedef union { + bool b; + float flt; + double dbl; + int32_t i32; + int64_t i64; + uint32_t u32; + uint64_t u64; + const upb_map* map; + const upb_msg* msg; + const upb_array* arr; + const void* ptr; + upb_strview str; +} upb_msgval; + +#define ACCESSORS(name, membername, ctype) \ + UPB_INLINE ctype upb_msgval_get ## name(upb_msgval v) { \ + return v.membername; \ + } \ + UPB_INLINE void upb_msgval_set ## name(upb_msgval *v, ctype cval) { \ + v->membername = cval; \ + } \ + UPB_INLINE upb_msgval upb_msgval_ ## name(ctype v) { \ + upb_msgval ret; \ + ret.membername = v; \ + return ret; \ + } + +ACCESSORS(bool, b, bool) +ACCESSORS(float, flt, float) +ACCESSORS(double, dbl, double) +ACCESSORS(int32, i32, int32_t) +ACCESSORS(int64, i64, int64_t) +ACCESSORS(uint32, u32, uint32_t) +ACCESSORS(uint64, u64, uint64_t) +ACCESSORS(map, map, const upb_map*) +ACCESSORS(msg, msg, const upb_msg*) +ACCESSORS(ptr, ptr, const void*) +ACCESSORS(arr, arr, const upb_array*) +ACCESSORS(str, str, upb_strview) + +#undef ACCESSORS + +UPB_INLINE upb_msgval upb_msgval_makestr(const char *data, size_t size) { + return upb_msgval_str(upb_strview_make(data, size)); +} + +/** upb_msg *******************************************************************/ + +/* A upb_msg represents a protobuf message. It always corresponds to a specific + * upb_msglayout, which describes how it is laid out in memory. */ + +/* Read-only message API. Can be safely called by anyone. */ + +/* Returns the value associated with this field: + * - for scalar fields (including strings), the value directly. + * - return upb_msg*, or upb_map* for msg/map. + * If the field is unset for these field types, returns NULL. + * + * TODO(haberman): should we let users store cached array/map/msg + * pointers here for fields that are unset? Could be useful for the + * strongly-owned submessage model (ie. generated C API that doesn't use + * arenas). + */ +upb_msgval upb_msg_get(const upb_msg *msg, + int field_index, + const upb_msglayout *l); + +/* May only be called for fields where upb_fielddef_haspresence(f) == true. */ +bool upb_msg_has(const upb_msg *msg, + int field_index, + const upb_msglayout *l); + +/* Mutable message API. May only be called by the owner of the message who + * knows its ownership scheme and how to keep it consistent. */ + +/* Sets the given field to the given value. Does not perform any memory + * management: if you overwrite a pointer to a msg/array/map/string without + * cleaning it up (or using an arena) it will leak. + */ +void upb_msg_set(upb_msg *msg, + int field_index, + upb_msgval val, + const upb_msglayout *l); + +/* For a primitive field, set it back to its default. For repeated, string, and + * submessage fields set it back to NULL. This could involve releasing some + * internal memory (for example, from an extension dictionary), but it is not + * recursive in any way and will not recover any memory that may be used by + * arrays/maps/strings/msgs that this field may have pointed to. + */ +bool upb_msg_clearfield(upb_msg *msg, + int field_index, + const upb_msglayout *l); + +/* TODO(haberman): copyfrom()/mergefrom()? */ + +/** upb_array *****************************************************************/ + +/* A upb_array stores data for a repeated field. The memory management + * semantics are the same as upb_msg. A upb_array allocates dynamic + * memory internally for the array elements. */ + +upb_fieldtype_t upb_array_type(const upb_array *arr); + +/* Read-only interface. Safe for anyone to call. */ + +size_t upb_array_size(const upb_array *arr); +upb_msgval upb_array_get(const upb_array *arr, upb_fieldtype_t type, size_t i); + +/* Write interface. May only be called by the message's owner who can enforce + * its memory management invariants. */ + +bool upb_array_set(upb_array *arr, upb_fieldtype_t type, size_t i, + upb_msgval val, upb_arena *arena); + +/** upb_map *******************************************************************/ + +/* A upb_map stores data for a map field. The memory management semantics are + * the same as upb_msg, with one notable exception. upb_map will internally + * store a copy of all string keys, but *not* any string values or submessages. + * So you must ensure that any string or message values outlive the map, and you + * must delete them manually when they are no longer required. */ + +upb_map *upb_map_new(upb_fieldtype_t ktype, upb_fieldtype_t vtype, + upb_arena *a); + +/* Read-only interface. Safe for anyone to call. */ + +size_t upb_map_size(const upb_map *map); +upb_fieldtype_t upb_map_keytype(const upb_map *map); +upb_fieldtype_t upb_map_valuetype(const upb_map *map); +bool upb_map_get(const upb_map *map, upb_msgval key, upb_msgval *val); + +/* Write interface. May only be called by the message's owner who can enforce + * its memory management invariants. */ + +/* Sets or overwrites an entry in the map. Return value indicates whether + * the operation succeeded or failed with OOM, and also whether an existing + * key was replaced or not. */ +bool upb_map_set(upb_map *map, + upb_msgval key, upb_msgval val, + upb_msgval *valremoved); + +/* Deletes an entry in the map. Returns true if the key was present. */ +bool upb_map_del(upb_map *map, upb_msgval key); + +/** upb_mapiter ***************************************************************/ + +/* For iterating over a map. Map iterators are invalidated by mutations to the + * map, but an invalidated iterator will never return junk or crash the process. + * An invalidated iterator may return entries that were already returned though, + * and if you keep invalidating the iterator during iteration, the program may + * enter an infinite loop. */ + +size_t upb_mapiter_sizeof(void); + +void upb_mapiter_begin(upb_mapiter *i, const upb_map *t); +upb_mapiter *upb_mapiter_new(const upb_map *t, upb_alloc *a); +void upb_mapiter_free(upb_mapiter *i, upb_alloc *a); +void upb_mapiter_next(upb_mapiter *i); +bool upb_mapiter_done(const upb_mapiter *i); + +upb_msgval upb_mapiter_key(const upb_mapiter *i); +upb_msgval upb_mapiter_value(const upb_mapiter *i); +void upb_mapiter_setdone(upb_mapiter *i); +bool upb_mapiter_isequal(const upb_mapiter *i1, const upb_mapiter *i2); + +#include "upb/port_undef.inc" + +#endif /* UPB_LEGACY_MSG_REFLECTION_H_ */ diff --git a/upb/msg.c b/upb/msg.c new file mode 100644 index 00000000000..a77da5665c7 --- /dev/null +++ b/upb/msg.c @@ -0,0 +1,111 @@ + +#include "upb/msg.h" + +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +#define VOIDPTR_AT(msg, ofs) (void*)((char*)msg + (int)ofs) + +/* Internal members of a upb_msg. We can change this without breaking binary + * compatibility. We put these before the user's data. The user's upb_msg* + * points after the upb_msg_internal. */ + +/* Used when a message is not extendable. */ +typedef struct { + char *unknown; + size_t unknown_len; + size_t unknown_size; +} upb_msg_internal; + +/* Used when a message is extendable. */ +typedef struct { + upb_inttable *extdict; + upb_msg_internal base; +} upb_msg_internal_withext; + +static int upb_msg_internalsize(const upb_msglayout *l) { + return sizeof(upb_msg_internal) - l->extendable * sizeof(void *); +} + +static size_t upb_msg_sizeof(const upb_msglayout *l) { + return l->size + upb_msg_internalsize(l); +} + +static upb_msg_internal *upb_msg_getinternal(upb_msg *msg) { + return VOIDPTR_AT(msg, -sizeof(upb_msg_internal)); +} + +static const upb_msg_internal *upb_msg_getinternal_const(const upb_msg *msg) { + return VOIDPTR_AT(msg, -sizeof(upb_msg_internal)); +} + +static upb_msg_internal_withext *upb_msg_getinternalwithext( + upb_msg *msg, const upb_msglayout *l) { + UPB_ASSERT(l->extendable); + return VOIDPTR_AT(msg, -sizeof(upb_msg_internal_withext)); +} + +upb_msg *upb_msg_new(const upb_msglayout *l, upb_arena *a) { + upb_alloc *alloc = upb_arena_alloc(a); + void *mem = upb_malloc(alloc, upb_msg_sizeof(l)); + upb_msg_internal *in; + upb_msg *msg; + + if (!mem) { + return NULL; + } + + msg = VOIDPTR_AT(mem, upb_msg_internalsize(l)); + + /* Initialize normal members. */ + memset(msg, 0, l->size); + + /* Initialize internal members. */ + in = upb_msg_getinternal(msg); + in->unknown = NULL; + in->unknown_len = 0; + in->unknown_size = 0; + + if (l->extendable) { + upb_msg_getinternalwithext(msg, l)->extdict = NULL; + } + + return msg; +} + +upb_array *upb_array_new(upb_arena *a) { + upb_array *ret = upb_arena_malloc(a, sizeof(upb_array)); + + if (!ret) { + return NULL; + } + + ret->data = NULL; + ret->len = 0; + ret->size = 0; + + return ret; +} + +void upb_msg_addunknown(upb_msg *msg, const char *data, size_t len, + upb_arena *arena) { + upb_msg_internal *in = upb_msg_getinternal(msg); + if (len > in->unknown_size - in->unknown_len) { + upb_alloc *alloc = upb_arena_alloc(arena); + size_t need = in->unknown_size + len; + size_t newsize = UPB_MAX(in->unknown_size * 2, need); + in->unknown = upb_realloc(alloc, in->unknown, in->unknown_size, newsize); + in->unknown_size = newsize; + } + memcpy(in->unknown + in->unknown_len, data, len); + in->unknown_len += len; +} + +const char *upb_msg_getunknown(const upb_msg *msg, size_t *len) { + const upb_msg_internal* in = upb_msg_getinternal_const(msg); + *len = in->unknown_len; + return in->unknown; +} + +#undef VOIDPTR_AT diff --git a/upb/msg.h b/upb/msg.h new file mode 100644 index 00000000000..4bec023bd3e --- /dev/null +++ b/upb/msg.h @@ -0,0 +1,69 @@ +/* +** Data structures for message tables, used for parsing and serialization. +** This are much lighter-weight than full reflection, but they are do not +** have enough information to convert to text format, JSON, etc. +** +** The definitions in this file are internal to upb. +**/ + +#ifndef UPB_MSG_H_ +#define UPB_MSG_H_ + +#include +#include +#include "upb/upb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void upb_msg; + +/** upb_msglayout *************************************************************/ + +/* upb_msglayout represents the memory layout of a given upb_msgdef. The + * members are public so generated code can initialize them, but users MUST NOT + * read or write any of its members. */ + +typedef struct { + uint32_t number; + uint16_t offset; + int16_t presence; /* If >0, hasbit_index+1. If <0, oneof_index+1. */ + uint16_t submsg_index; /* undefined if descriptortype != MESSAGE or GROUP. */ + uint8_t descriptortype; + uint8_t label; +} upb_msglayout_field; + +typedef struct upb_msglayout { + const struct upb_msglayout *const* submsgs; + const upb_msglayout_field *fields; + /* Must be aligned to sizeof(void*). Doesn't include internal members like + * unknown fields, extension dict, pointer to msglayout, etc. */ + uint16_t size; + uint16_t field_count; + bool extendable; +} upb_msglayout; + +/** Message internal representation *******************************************/ + +/* Our internal representation for repeated fields. */ +typedef struct { + void *data; /* Each element is element_size. */ + size_t len; /* Measured in elements. */ + size_t size; /* Measured in elements. */ +} upb_array; + +upb_msg *upb_msg_new(const upb_msglayout *l, upb_arena *a); +upb_msg *upb_msg_new(const upb_msglayout *l, upb_arena *a); + +void upb_msg_addunknown(upb_msg *msg, const char *data, size_t len, + upb_arena *arena); +const char *upb_msg_getunknown(const upb_msg *msg, size_t *len); + +upb_array *upb_array_new(upb_arena *a); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_MSG_H_ */ diff --git a/upb/msgfactory.c b/upb/msgfactory.c new file mode 100644 index 00000000000..4ecf2725b15 --- /dev/null +++ b/upb/msgfactory.c @@ -0,0 +1,248 @@ + +#include "upb/msgfactory.h" + +#include "upb/port_def.inc" + +static bool is_power_of_two(size_t val) { + return (val & (val - 1)) == 0; +} + +/* Align up to the given power of 2. */ +static size_t align_up(size_t val, size_t align) { + UPB_ASSERT(is_power_of_two(align)); + return (val + align - 1) & ~(align - 1); +} + +static size_t div_round_up(size_t n, size_t d) { + return (n + d - 1) / d; +} + +static size_t upb_msgval_sizeof2(upb_fieldtype_t type) { + switch (type) { + case UPB_TYPE_DOUBLE: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + return 8; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_FLOAT: + return 4; + case UPB_TYPE_BOOL: + return 1; + case UPB_TYPE_MESSAGE: + return sizeof(void*); + case UPB_TYPE_BYTES: + case UPB_TYPE_STRING: + return sizeof(upb_strview); + } + UPB_UNREACHABLE(); +} + +static uint8_t upb_msg_fielddefsize(const upb_fielddef *f) { + if (upb_fielddef_isseq(f)) { + return sizeof(void*); + } else { + return upb_msgval_sizeof2(upb_fielddef_type(f)); + } +} + + +/** upb_msglayout *************************************************************/ + +static void upb_msglayout_free(upb_msglayout *l) { + upb_gfree(l); +} + +static size_t upb_msglayout_place(upb_msglayout *l, size_t size) { + size_t ret; + + l->size = align_up(l->size, size); + ret = l->size; + l->size += size; + return ret; +} + +static bool upb_msglayout_init(const upb_msgdef *m, + upb_msglayout *l, + upb_msgfactory *factory) { + upb_msg_field_iter it; + upb_msg_oneof_iter oit; + size_t hasbit; + size_t submsg_count = 0; + const upb_msglayout **submsgs; + upb_msglayout_field *fields; + + for (upb_msg_field_begin(&it, m); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + const upb_fielddef* f = upb_msg_iter_field(&it); + if (upb_fielddef_issubmsg(f)) { + submsg_count++; + } + } + + memset(l, 0, sizeof(*l)); + + fields = upb_gmalloc(upb_msgdef_numfields(m) * sizeof(*fields)); + submsgs = upb_gmalloc(submsg_count * sizeof(*submsgs)); + + if ((!fields && upb_msgdef_numfields(m)) || + (!submsgs && submsg_count)) { + /* OOM. */ + upb_gfree(fields); + upb_gfree(submsgs); + return false; + } + + l->field_count = upb_msgdef_numfields(m); + l->fields = fields; + l->submsgs = submsgs; + + /* Allocate data offsets in three stages: + * + * 1. hasbits. + * 2. regular fields. + * 3. oneof fields. + * + * OPT: There is a lot of room for optimization here to minimize the size. + */ + + /* Allocate hasbits and set basic field attributes. */ + submsg_count = 0; + for (upb_msg_field_begin(&it, m), hasbit = 0; + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + const upb_fielddef* f = upb_msg_iter_field(&it); + upb_msglayout_field *field = &fields[upb_fielddef_index(f)]; + + field->number = upb_fielddef_number(f); + field->descriptortype = upb_fielddef_descriptortype(f); + field->label = upb_fielddef_label(f); + + if (upb_fielddef_issubmsg(f)) { + const upb_msglayout *sub_layout = + upb_msgfactory_getlayout(factory, upb_fielddef_msgsubdef(f)); + field->submsg_index = submsg_count++; + submsgs[field->submsg_index] = sub_layout; + } + + if (upb_fielddef_haspresence(f) && !upb_fielddef_containingoneof(f)) { + field->presence = (hasbit++); + } else { + field->presence = 0; + } + } + + /* Account for space used by hasbits. */ + l->size = div_round_up(hasbit, 8); + + /* Allocate non-oneof fields. */ + for (upb_msg_field_begin(&it, m); !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + const upb_fielddef* f = upb_msg_iter_field(&it); + size_t field_size = upb_msg_fielddefsize(f); + size_t index = upb_fielddef_index(f); + + if (upb_fielddef_containingoneof(f)) { + /* Oneofs are handled separately below. */ + continue; + } + + fields[index].offset = upb_msglayout_place(l, field_size); + } + + /* Allocate oneof fields. Each oneof field consists of a uint32 for the case + * and space for the actual data. */ + for (upb_msg_oneof_begin(&oit, m); !upb_msg_oneof_done(&oit); + upb_msg_oneof_next(&oit)) { + const upb_oneofdef* o = upb_msg_iter_oneof(&oit); + upb_oneof_iter fit; + + size_t case_size = sizeof(uint32_t); /* Could potentially optimize this. */ + size_t field_size = 0; + uint32_t case_offset; + uint32_t data_offset; + + /* Calculate field size: the max of all field sizes. */ + for (upb_oneof_begin(&fit, o); + !upb_oneof_done(&fit); + upb_oneof_next(&fit)) { + const upb_fielddef* f = upb_oneof_iter_field(&fit); + field_size = UPB_MAX(field_size, upb_msg_fielddefsize(f)); + } + + /* Align and allocate case offset. */ + case_offset = upb_msglayout_place(l, case_size); + data_offset = upb_msglayout_place(l, field_size); + + for (upb_oneof_begin(&fit, o); + !upb_oneof_done(&fit); + upb_oneof_next(&fit)) { + const upb_fielddef* f = upb_oneof_iter_field(&fit); + fields[upb_fielddef_index(f)].offset = data_offset; + fields[upb_fielddef_index(f)].presence = ~case_offset; + } + } + + /* Size of the entire structure should be a multiple of its greatest + * alignment. TODO: track overall alignment for real? */ + l->size = align_up(l->size, 8); + + return true; +} + + +/** upb_msgfactory ************************************************************/ + +struct upb_msgfactory { + const upb_symtab *symtab; /* We own a ref. */ + upb_inttable layouts; +}; + +upb_msgfactory *upb_msgfactory_new(const upb_symtab *symtab) { + upb_msgfactory *ret = upb_gmalloc(sizeof(*ret)); + + ret->symtab = symtab; + upb_inttable_init(&ret->layouts, UPB_CTYPE_PTR); + + return ret; +} + +void upb_msgfactory_free(upb_msgfactory *f) { + upb_inttable_iter i; + upb_inttable_begin(&i, &f->layouts); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_msglayout *l = upb_value_getptr(upb_inttable_iter_value(&i)); + upb_msglayout_free(l); + } + + upb_inttable_uninit(&f->layouts); + upb_gfree(f); +} + +const upb_symtab *upb_msgfactory_symtab(const upb_msgfactory *f) { + return f->symtab; +} + +const upb_msglayout *upb_msgfactory_getlayout(upb_msgfactory *f, + const upb_msgdef *m) { + upb_value v; + UPB_ASSERT(upb_symtab_lookupmsg(f->symtab, upb_msgdef_fullname(m)) == m); + UPB_ASSERT(!upb_msgdef_mapentry(m)); + + if (upb_inttable_lookupptr(&f->layouts, m, &v)) { + UPB_ASSERT(upb_value_getptr(v)); + return upb_value_getptr(v); + } else { + /* In case of circular dependency, layout has to be inserted first. */ + upb_msglayout *l = upb_gmalloc(sizeof(*l)); + upb_msgfactory *mutable_f = (void*)f; + upb_inttable_insertptr(&mutable_f->layouts, m, upb_value_ptr(l)); + UPB_ASSERT(l); + if (!upb_msglayout_init(m, l, f)) { + upb_msglayout_free(l); + } + return l; + } +} diff --git a/upb/msgfactory.h b/upb/msgfactory.h new file mode 100644 index 00000000000..9b3b5999389 --- /dev/null +++ b/upb/msgfactory.h @@ -0,0 +1,48 @@ + +#include "upb/def.h" +#include "upb/msg.h" + +#ifndef UPB_MSGFACTORY_H_ +#define UPB_MSGFACTORY_H_ + +/** upb_msgfactory ************************************************************/ + +struct upb_msgfactory; +typedef struct upb_msgfactory upb_msgfactory; + +#ifdef __cplusplus +extern "C" { +#endif + +/* A upb_msgfactory contains a cache of upb_msglayout, upb_handlers, and + * upb_visitorplan objects. These are the objects necessary to represent, + * populate, and and visit upb_msg objects. + * + * These caches are all populated by upb_msgdef, and lazily created on demand. + */ + +/* Creates and destroys a msgfactory, respectively. The messages for this + * msgfactory must come from |symtab| (which should outlive the msgfactory). */ +upb_msgfactory *upb_msgfactory_new(const upb_symtab *symtab); +void upb_msgfactory_free(upb_msgfactory *f); + +const upb_symtab *upb_msgfactory_symtab(const upb_msgfactory *f); + +/* The functions to get cached objects, lazily creating them on demand. These + * all require: + * + * - m is in upb_msgfactory_symtab(f) + * - upb_msgdef_mapentry(m) == false (since map messages can't have layouts). + * + * The returned objects will live for as long as the msgfactory does. + * + * TODO(haberman): consider making this thread-safe and take a const + * upb_msgfactory. */ +const upb_msglayout *upb_msgfactory_getlayout(upb_msgfactory *f, + const upb_msgdef *m); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* UPB_MSGFACTORY_H_ */ diff --git a/upb/pb/compile_decoder.c b/upb/pb/compile_decoder.c new file mode 100644 index 00000000000..f5c7d656466 --- /dev/null +++ b/upb/pb/compile_decoder.c @@ -0,0 +1,919 @@ +/* +** protobuf decoder bytecode compiler +** +** Code to compile a upb::Handlers into bytecode for decoding a protobuf +** according to that specific schema and destination handlers. +** +** Bytecode definition is in decoder.int.h. +*/ + +#include +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" + +#ifdef UPB_DUMP_BYTECODE +#include +#endif + +#include "upb/port_def.inc" + +#define MAXLABEL 5 +#define EMPTYLABEL -1 + +/* upb_pbdecodermethod ********************************************************/ + +static void freemethod(upb_pbdecodermethod *method) { + upb_inttable_uninit(&method->dispatch); + upb_gfree(method); +} + +static upb_pbdecodermethod *newmethod(const upb_handlers *dest_handlers, + mgroup *group) { + upb_pbdecodermethod *ret = upb_gmalloc(sizeof(*ret)); + upb_byteshandler_init(&ret->input_handler_); + + ret->group = group; + ret->dest_handlers_ = dest_handlers; + upb_inttable_init(&ret->dispatch, UPB_CTYPE_UINT64); + + return ret; +} + +const upb_handlers *upb_pbdecodermethod_desthandlers( + const upb_pbdecodermethod *m) { + return m->dest_handlers_; +} + +const upb_byteshandler *upb_pbdecodermethod_inputhandler( + const upb_pbdecodermethod *m) { + return &m->input_handler_; +} + +bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m) { + return m->is_native_; +} + + +/* mgroup *********************************************************************/ + +static void freegroup(mgroup *g) { + upb_inttable_iter i; + + upb_inttable_begin(&i, &g->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + freemethod(upb_value_getptr(upb_inttable_iter_value(&i))); + } + + upb_inttable_uninit(&g->methods); + upb_gfree(g->bytecode); + upb_gfree(g); +} + +mgroup *newgroup(void) { + mgroup *g = upb_gmalloc(sizeof(*g)); + upb_inttable_init(&g->methods, UPB_CTYPE_PTR); + g->bytecode = NULL; + g->bytecode_end = NULL; + return g; +} + + +/* bytecode compiler **********************************************************/ + +/* Data used only at compilation time. */ +typedef struct { + mgroup *group; + + uint32_t *pc; + int fwd_labels[MAXLABEL]; + int back_labels[MAXLABEL]; + + /* For fields marked "lazy", parse them lazily or eagerly? */ + bool lazy; +} compiler; + +static compiler *newcompiler(mgroup *group, bool lazy) { + compiler *ret = upb_gmalloc(sizeof(*ret)); + int i; + + ret->group = group; + ret->lazy = lazy; + for (i = 0; i < MAXLABEL; i++) { + ret->fwd_labels[i] = EMPTYLABEL; + ret->back_labels[i] = EMPTYLABEL; + } + return ret; +} + +static void freecompiler(compiler *c) { + upb_gfree(c); +} + +const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); + +/* How many words an instruction is. */ +static int instruction_len(uint32_t instr) { + switch (getop(instr)) { + case OP_SETDISPATCH: return 1 + ptr_words; + case OP_TAGN: return 3; + case OP_SETBIGGROUPNUM: return 2; + default: return 1; + } +} + +bool op_has_longofs(int32_t instruction) { + switch (getop(instruction)) { + case OP_CALL: + case OP_BRANCH: + case OP_CHECKDELIM: + return true; + /* The "tag" instructions only have 8 bytes available for the jump target, + * but that is ok because these opcodes only require short jumps. */ + case OP_TAG1: + case OP_TAG2: + case OP_TAGN: + return false; + default: + UPB_ASSERT(false); + return false; + } +} + +static int32_t getofs(uint32_t instruction) { + if (op_has_longofs(instruction)) { + return (int32_t)instruction >> 8; + } else { + return (int8_t)(instruction >> 8); + } +} + +static void setofs(uint32_t *instruction, int32_t ofs) { + if (op_has_longofs(*instruction)) { + *instruction = getop(*instruction) | (uint32_t)ofs << 8; + } else { + *instruction = (*instruction & ~0xff00) | ((ofs & 0xff) << 8); + } + UPB_ASSERT(getofs(*instruction) == ofs); /* Would fail in cases of overflow. */ +} + +static uint32_t pcofs(compiler *c) { return c->pc - c->group->bytecode; } + +/* Defines a local label at the current PC location. All previous forward + * references are updated to point to this location. The location is noted + * for any future backward references. */ +static void label(compiler *c, unsigned int label) { + int val; + uint32_t *codep; + + UPB_ASSERT(label < MAXLABEL); + val = c->fwd_labels[label]; + codep = (val == EMPTYLABEL) ? NULL : c->group->bytecode + val; + while (codep) { + int ofs = getofs(*codep); + setofs(codep, c->pc - codep - instruction_len(*codep)); + codep = ofs ? codep + ofs : NULL; + } + c->fwd_labels[label] = EMPTYLABEL; + c->back_labels[label] = pcofs(c); +} + +/* Creates a reference to a numbered label; either a forward reference + * (positive arg) or backward reference (negative arg). For forward references + * the value returned now is actually a "next" pointer into a linked list of all + * instructions that use this label and will be patched later when the label is + * defined with label(). + * + * The returned value is the offset that should be written into the instruction. + */ +static int32_t labelref(compiler *c, int label) { + UPB_ASSERT(label < MAXLABEL); + if (label == LABEL_DISPATCH) { + /* No resolving required. */ + return 0; + } else if (label < 0) { + /* Backward local label. Relative to the next instruction. */ + uint32_t from = (c->pc + 1) - c->group->bytecode; + return c->back_labels[-label] - from; + } else { + /* Forward local label: prepend to (possibly-empty) linked list. */ + int *lptr = &c->fwd_labels[label]; + int32_t ret = (*lptr == EMPTYLABEL) ? 0 : *lptr - pcofs(c); + *lptr = pcofs(c); + return ret; + } +} + +static void put32(compiler *c, uint32_t v) { + mgroup *g = c->group; + if (c->pc == g->bytecode_end) { + int ofs = pcofs(c); + size_t oldsize = g->bytecode_end - g->bytecode; + size_t newsize = UPB_MAX(oldsize * 2, 64); + /* TODO(haberman): handle OOM. */ + g->bytecode = upb_grealloc(g->bytecode, oldsize * sizeof(uint32_t), + newsize * sizeof(uint32_t)); + g->bytecode_end = g->bytecode + newsize; + c->pc = g->bytecode + ofs; + } + *c->pc++ = v; +} + +static void putop(compiler *c, int op, ...) { + va_list ap; + va_start(ap, op); + + switch (op) { + case OP_SETDISPATCH: { + uintptr_t ptr = (uintptr_t)va_arg(ap, void*); + put32(c, OP_SETDISPATCH); + put32(c, ptr); + if (sizeof(uintptr_t) > sizeof(uint32_t)) + put32(c, (uint64_t)ptr >> 32); + break; + } + case OP_STARTMSG: + case OP_ENDMSG: + case OP_PUSHLENDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_HALT: + case OP_RET: + case OP_DISPATCH: + put32(c, op); + break; + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_STRING: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + put32(c, op | va_arg(ap, upb_selector_t) << 8); + break; + case OP_SETBIGGROUPNUM: + put32(c, op); + put32(c, va_arg(ap, int)); + break; + case OP_CALL: { + const upb_pbdecodermethod *method = va_arg(ap, upb_pbdecodermethod *); + put32(c, op | (method->code_base.ofs - (pcofs(c) + 1)) << 8); + break; + } + case OP_CHECKDELIM: + case OP_BRANCH: { + uint32_t instruction = op; + int label = va_arg(ap, int); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + break; + } + case OP_TAG1: + case OP_TAG2: { + int label = va_arg(ap, int); + uint64_t tag = va_arg(ap, uint64_t); + uint32_t instruction = op | (tag << 16); + UPB_ASSERT(tag <= 0xffff); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + break; + } + case OP_TAGN: { + int label = va_arg(ap, int); + uint64_t tag = va_arg(ap, uint64_t); + uint32_t instruction = op | (upb_value_size(tag) << 16); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + put32(c, tag); + put32(c, tag >> 32); + break; + } + } + + va_end(ap); +} + +#if defined(UPB_DUMP_BYTECODE) + +const char *upb_pbdecoder_getopname(unsigned int op) { +#define QUOTE(x) #x +#define EXPAND_AND_QUOTE(x) QUOTE(x) +#define OPNAME(x) OP_##x +#define OP(x) case OPNAME(x): return EXPAND_AND_QUOTE(OPNAME(x)); +#define T(x) OP(PARSE_##x) + /* Keep in sync with list in decoder.int.h. */ + switch ((opcode)op) { + T(DOUBLE) T(FLOAT) T(INT64) T(UINT64) T(INT32) T(FIXED64) T(FIXED32) + T(BOOL) T(UINT32) T(SFIXED32) T(SFIXED64) T(SINT32) T(SINT64) + OP(STARTMSG) OP(ENDMSG) OP(STARTSEQ) OP(ENDSEQ) OP(STARTSUBMSG) + OP(ENDSUBMSG) OP(STARTSTR) OP(STRING) OP(ENDSTR) OP(CALL) OP(RET) + OP(PUSHLENDELIM) OP(PUSHTAGDELIM) OP(SETDELIM) OP(CHECKDELIM) + OP(BRANCH) OP(TAG1) OP(TAG2) OP(TAGN) OP(SETDISPATCH) OP(POP) + OP(SETBIGGROUPNUM) OP(DISPATCH) OP(HALT) + } + return ""; +#undef OP +#undef T +} + +#endif + +#ifdef UPB_DUMP_BYTECODE + +static void dumpbc(uint32_t *p, uint32_t *end, FILE *f) { + + uint32_t *begin = p; + + while (p < end) { + fprintf(f, "%p %8tx", p, p - begin); + uint32_t instr = *p++; + uint8_t op = getop(instr); + fprintf(f, " %s", upb_pbdecoder_getopname(op)); + switch ((opcode)op) { + case OP_SETDISPATCH: { + const upb_inttable *dispatch; + memcpy(&dispatch, p, sizeof(void*)); + p += ptr_words; + const upb_pbdecodermethod *method = + (void *)((char *)dispatch - + offsetof(upb_pbdecodermethod, dispatch)); + fprintf(f, " %s", upb_msgdef_fullname( + upb_handlers_msgdef(method->dest_handlers_))); + break; + } + case OP_DISPATCH: + case OP_STARTMSG: + case OP_ENDMSG: + case OP_PUSHLENDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_HALT: + case OP_RET: + break; + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_STRING: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + fprintf(f, " %d", instr >> 8); + break; + case OP_SETBIGGROUPNUM: + fprintf(f, " %d", *p++); + break; + case OP_CHECKDELIM: + case OP_CALL: + case OP_BRANCH: + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + break; + case OP_TAG1: + case OP_TAG2: { + fprintf(f, " tag:0x%x", instr >> 16); + if (getofs(instr)) { + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + } + break; + } + case OP_TAGN: { + uint64_t tag = *p++; + tag |= (uint64_t)*p++ << 32; + fprintf(f, " tag:0x%llx", (long long)tag); + fprintf(f, " n:%d", instr >> 16); + if (getofs(instr)) { + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + } + break; + } + } + fputs("\n", f); + } +} + +#endif + +static uint64_t get_encoded_tag(const upb_fielddef *f, int wire_type) { + uint32_t tag = (upb_fielddef_number(f) << 3) | wire_type; + uint64_t encoded_tag = upb_vencode32(tag); + /* No tag should be greater than 5 bytes. */ + UPB_ASSERT(encoded_tag <= 0xffffffffff); + return encoded_tag; +} + +static void putchecktag(compiler *c, const upb_fielddef *f, + int wire_type, int dest) { + uint64_t tag = get_encoded_tag(f, wire_type); + switch (upb_value_size(tag)) { + case 1: + putop(c, OP_TAG1, dest, tag); + break; + case 2: + putop(c, OP_TAG2, dest, tag); + break; + default: + putop(c, OP_TAGN, dest, tag); + break; + } +} + +static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) { + upb_selector_t selector; + bool ok = upb_handlers_getselector(f, type, &selector); + UPB_ASSERT(ok); + return selector; +} + +/* Takes an existing, primary dispatch table entry and repacks it with a + * different alternate wire type. Called when we are inserting a secondary + * dispatch table entry for an alternate wire type. */ +static uint64_t repack(uint64_t dispatch, int new_wt2) { + uint64_t ofs; + uint8_t wt1; + uint8_t old_wt2; + upb_pbdecoder_unpackdispatch(dispatch, &ofs, &wt1, &old_wt2); + UPB_ASSERT(old_wt2 == NO_WIRE_TYPE); /* wt2 should not be set yet. */ + return upb_pbdecoder_packdispatch(ofs, wt1, new_wt2); +} + +/* Marks the current bytecode position as the dispatch target for this message, + * field, and wire type. */ +static void dispatchtarget(compiler *c, upb_pbdecodermethod *method, + const upb_fielddef *f, int wire_type) { + /* Offset is relative to msg base. */ + uint64_t ofs = pcofs(c) - method->code_base.ofs; + uint32_t fn = upb_fielddef_number(f); + upb_inttable *d = &method->dispatch; + upb_value v; + if (upb_inttable_remove(d, fn, &v)) { + /* TODO: prioritize based on packed setting in .proto file. */ + uint64_t repacked = repack(upb_value_getuint64(v), wire_type); + upb_inttable_insert(d, fn, upb_value_uint64(repacked)); + upb_inttable_insert(d, fn + UPB_MAX_FIELDNUMBER, upb_value_uint64(ofs)); + } else { + uint64_t val = upb_pbdecoder_packdispatch(ofs, wire_type, NO_WIRE_TYPE); + upb_inttable_insert(d, fn, upb_value_uint64(val)); + } +} + +static void putpush(compiler *c, const upb_fielddef *f) { + if (upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE) { + putop(c, OP_PUSHLENDELIM); + } else { + uint32_t fn = upb_fielddef_number(f); + if (fn >= 1 << 24) { + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_SETBIGGROUPNUM, fn); + } else { + putop(c, OP_PUSHTAGDELIM, fn); + } + } +} + +static upb_pbdecodermethod *find_submethod(const compiler *c, + const upb_pbdecodermethod *method, + const upb_fielddef *f) { + const upb_handlers *sub = + upb_handlers_getsubhandlers(method->dest_handlers_, f); + upb_value v; + return upb_inttable_lookupptr(&c->group->methods, sub, &v) + ? upb_value_getptr(v) + : NULL; +} + +static void putsel(compiler *c, opcode op, upb_selector_t sel, + const upb_handlers *h) { + if (upb_handlers_gethandler(h, sel, NULL)) { + putop(c, op, sel); + } +} + +/* Puts an opcode to call a callback, but only if a callback actually exists for + * this field and handler type. */ +static void maybeput(compiler *c, opcode op, const upb_handlers *h, + const upb_fielddef *f, upb_handlertype_t type) { + putsel(c, op, getsel(f, type), h); +} + +static bool haslazyhandlers(const upb_handlers *h, const upb_fielddef *f) { + if (!upb_fielddef_lazy(f)) + return false; + + return upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STARTSTR), NULL) || + upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STRING), NULL) || + upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_ENDSTR), NULL); +} + + +/* bytecode compiler code generation ******************************************/ + +/* Symbolic names for our local labels. */ +#define LABEL_LOOPSTART 1 /* Top of a repeated field loop. */ +#define LABEL_LOOPBREAK 2 /* To jump out of a repeated loop */ +#define LABEL_FIELD 3 /* Jump backward to find the most recent field. */ +#define LABEL_ENDMSG 4 /* To reach the OP_ENDMSG instr for this msg. */ + +/* Generates bytecode to parse a single non-lazy message field. */ +static void generate_msgfield(compiler *c, const upb_fielddef *f, + upb_pbdecodermethod *method) { + const upb_handlers *h = upb_pbdecodermethod_desthandlers(method); + const upb_pbdecodermethod *sub_m = find_submethod(c, method, f); + int wire_type; + + if (!sub_m) { + /* Don't emit any code for this field at all; it will be parsed as an + * unknown field. + * + * TODO(haberman): we should change this to parse it as a string field + * instead. It will probably be faster, but more importantly, once we + * start vending unknown fields, a field shouldn't be treated as unknown + * just because it doesn't have subhandlers registered. */ + return; + } + + label(c, LABEL_FIELD); + + wire_type = + (upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE) + ? UPB_WIRE_TYPE_DELIMITED + : UPB_WIRE_TYPE_START_GROUP; + + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); + label(c, LABEL_LOOPSTART); + putpush(c, f); + putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG)); + putop(c, OP_CALL, sub_m); + putop(c, OP_POP); + maybeput(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG); + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + putop(c, OP_SETDELIM); + } + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, wire_type, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); + maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ); + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putpush(c, f); + putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG)); + putop(c, OP_CALL, sub_m); + putop(c, OP_POP); + maybeput(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG); + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + putop(c, OP_SETDELIM); + } + } +} + +/* Generates bytecode to parse a single string or lazy submessage field. */ +static void generate_delimfield(compiler *c, const upb_fielddef *f, + upb_pbdecodermethod *method) { + const upb_handlers *h = upb_pbdecodermethod_desthandlers(method); + + label(c, LABEL_FIELD); + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); + label(c, LABEL_LOOPSTART); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR)); + /* Need to emit even if no handler to skip past the string. */ + putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING)); + maybeput(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR); + putop(c, OP_POP); + putop(c, OP_SETDELIM); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); + maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ); + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR)); + putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING)); + maybeput(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR); + putop(c, OP_POP); + putop(c, OP_SETDELIM); + } +} + +/* Generates bytecode to parse a single primitive field. */ +static void generate_primitivefield(compiler *c, const upb_fielddef *f, + upb_pbdecodermethod *method) { + const upb_handlers *h = upb_pbdecodermethod_desthandlers(method); + upb_descriptortype_t descriptor_type = upb_fielddef_descriptortype(f); + opcode parse_type; + upb_selector_t sel; + int wire_type; + + label(c, LABEL_FIELD); + + /* From a decoding perspective, ENUM is the same as INT32. */ + if (descriptor_type == UPB_DESCRIPTOR_TYPE_ENUM) + descriptor_type = UPB_DESCRIPTOR_TYPE_INT32; + + parse_type = (opcode)descriptor_type; + + /* TODO(haberman): generate packed or non-packed first depending on "packed" + * setting in the fielddef. This will favor (in speed) whichever was + * specified. */ + + UPB_ASSERT((int)parse_type >= 0 && parse_type <= OP_MAX); + sel = getsel(f, upb_handlers_getprimitivehandlertype(f)); + wire_type = upb_pb_native_wire_types[upb_fielddef_descriptortype(f)]; + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); /* Packed */ + label(c, LABEL_LOOPSTART); + putop(c, parse_type, sel); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + dispatchtarget(c, method, f, wire_type); + putop(c, OP_PUSHTAGDELIM, 0); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); /* Non-packed */ + label(c, LABEL_LOOPSTART); + putop(c, parse_type, sel); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, wire_type, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); /* Packed and non-packed join. */ + maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ); + putop(c, OP_SETDELIM); /* Could remove for non-packed by dup ENDSEQ. */ + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putop(c, parse_type, sel); + } +} + +/* Adds bytecode for parsing the given message to the given decoderplan, + * while adding all dispatch targets to this message's dispatch table. */ +static void compile_method(compiler *c, upb_pbdecodermethod *method) { + const upb_handlers *h; + const upb_msgdef *md; + uint32_t* start_pc; + upb_msg_field_iter i; + upb_value val; + + UPB_ASSERT(method); + + /* Clear all entries in the dispatch table. */ + upb_inttable_uninit(&method->dispatch); + upb_inttable_init(&method->dispatch, UPB_CTYPE_UINT64); + + h = upb_pbdecodermethod_desthandlers(method); + md = upb_handlers_msgdef(h); + + method->code_base.ofs = pcofs(c); + putop(c, OP_SETDISPATCH, &method->dispatch); + putsel(c, OP_STARTMSG, UPB_STARTMSG_SELECTOR, h); + label(c, LABEL_FIELD); + start_pc = c->pc; + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_fieldtype_t type = upb_fielddef_type(f); + + if (type == UPB_TYPE_MESSAGE && !(haslazyhandlers(h, f) && c->lazy)) { + generate_msgfield(c, f, method); + } else if (type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES || + type == UPB_TYPE_MESSAGE) { + generate_delimfield(c, f, method); + } else { + generate_primitivefield(c, f, method); + } + } + + /* If there were no fields, or if no handlers were defined, we need to + * generate a non-empty loop body so that we can at least dispatch for unknown + * fields and check for the end of the message. */ + if (c->pc == start_pc) { + /* Check for end-of-message. */ + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + /* Unconditionally dispatch. */ + putop(c, OP_DISPATCH, 0); + } + + /* For now we just loop back to the last field of the message (or if none, + * the DISPATCH opcode for the message). */ + putop(c, OP_BRANCH, -LABEL_FIELD); + + /* Insert both a label and a dispatch table entry for this end-of-msg. */ + label(c, LABEL_ENDMSG); + val = upb_value_uint64(pcofs(c) - method->code_base.ofs); + upb_inttable_insert(&method->dispatch, DISPATCH_ENDMSG, val); + + putsel(c, OP_ENDMSG, UPB_ENDMSG_SELECTOR, h); + putop(c, OP_RET); + + upb_inttable_compact(&method->dispatch); +} + +/* Populate "methods" with new upb_pbdecodermethod objects reachable from "h". + * Returns the method for these handlers. + * + * Generates a new method for every destination handlers reachable from "h". */ +static void find_methods(compiler *c, const upb_handlers *h) { + upb_value v; + upb_msg_field_iter i; + const upb_msgdef *md; + upb_pbdecodermethod *method; + + if (upb_inttable_lookupptr(&c->group->methods, h, &v)) + return; + + method = newmethod(h, c->group); + upb_inttable_insertptr(&c->group->methods, h, upb_value_ptr(method)); + + /* Find submethods. */ + md = upb_handlers_msgdef(h); + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + const upb_handlers *sub_h; + if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE && + (sub_h = upb_handlers_getsubhandlers(h, f)) != NULL) { + /* We only generate a decoder method for submessages with handlers. + * Others will be parsed as unknown fields. */ + find_methods(c, sub_h); + } + } +} + +/* (Re-)compile bytecode for all messages in "msgs." + * Overwrites any existing bytecode in "c". */ +static void compile_methods(compiler *c) { + upb_inttable_iter i; + + /* Start over at the beginning of the bytecode. */ + c->pc = c->group->bytecode; + + upb_inttable_begin(&i, &c->group->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); + compile_method(c, method); + } +} + +static void set_bytecode_handlers(mgroup *g) { + upb_inttable_iter i; + upb_inttable_begin(&i, &g->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *m = upb_value_getptr(upb_inttable_iter_value(&i)); + upb_byteshandler *h = &m->input_handler_; + + m->code_base.ptr = g->bytecode + m->code_base.ofs; + + upb_byteshandler_setstartstr(h, upb_pbdecoder_startbc, m->code_base.ptr); + upb_byteshandler_setstring(h, upb_pbdecoder_decode, g); + upb_byteshandler_setendstr(h, upb_pbdecoder_end, m); + } +} + + +/* TODO(haberman): allow this to be constructed for an arbitrary set of dest + * handlers and other mgroups (but verify we have a transitive closure). */ +const mgroup *mgroup_new(const upb_handlers *dest, bool lazy) { + mgroup *g; + compiler *c; + + g = newgroup(); + c = newcompiler(g, lazy); + find_methods(c, dest); + + /* We compile in two passes: + * 1. all messages are assigned relative offsets from the beginning of the + * bytecode (saved in method->code_base). + * 2. forwards OP_CALL instructions can be correctly linked since message + * offsets have been previously assigned. + * + * Could avoid the second pass by linking OP_CALL instructions somehow. */ + compile_methods(c); + compile_methods(c); + g->bytecode_end = c->pc; + freecompiler(c); + +#ifdef UPB_DUMP_BYTECODE + { + FILE *f = fopen("/tmp/upb-bytecode", "w"); + UPB_ASSERT(f); + dumpbc(g->bytecode, g->bytecode_end, stderr); + dumpbc(g->bytecode, g->bytecode_end, f); + fclose(f); + + f = fopen("/tmp/upb-bytecode.bin", "wb"); + UPB_ASSERT(f); + fwrite(g->bytecode, 1, g->bytecode_end - g->bytecode, f); + fclose(f); + } +#endif + + set_bytecode_handlers(g); + return g; +} + + +/* upb_pbcodecache ************************************************************/ + +upb_pbcodecache *upb_pbcodecache_new(upb_handlercache *dest) { + upb_pbcodecache *c = upb_gmalloc(sizeof(*c)); + + if (!c) return NULL; + + c->dest = dest; + c->lazy = false; + + c->arena = upb_arena_new(); + if (!upb_inttable_init(&c->groups, UPB_CTYPE_CONSTPTR)) return NULL; + + return c; +} + +void upb_pbcodecache_free(upb_pbcodecache *c) { + upb_inttable_iter i; + + upb_inttable_begin(&i, &c->groups); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_value val = upb_inttable_iter_value(&i); + freegroup((void*)upb_value_getconstptr(val)); + } + + upb_inttable_uninit(&c->groups); + upb_arena_free(c->arena); + upb_gfree(c); +} + +void upb_pbdecodermethodopts_setlazy(upb_pbcodecache *c, bool lazy) { + UPB_ASSERT(upb_inttable_count(&c->groups) == 0); + c->lazy = lazy; +} + +const upb_pbdecodermethod *upb_pbcodecache_get(upb_pbcodecache *c, + const upb_msgdef *md) { + upb_value v; + bool ok; + const upb_handlers *h; + const mgroup *g; + + h = upb_handlercache_get(c->dest, md); + if (upb_inttable_lookupptr(&c->groups, md, &v)) { + g = upb_value_getconstptr(v); + } else { + g = mgroup_new(h, c->lazy); + ok = upb_inttable_insertptr(&c->groups, md, upb_value_constptr(g)); + UPB_ASSERT(ok); + } + + ok = upb_inttable_lookupptr(&g->methods, h, &v); + UPB_ASSERT(ok); + return upb_value_getptr(v); +} diff --git a/upb/pb/decoder.c b/upb/pb/decoder.c new file mode 100644 index 00000000000..735bef18f8e --- /dev/null +++ b/upb/pb/decoder.c @@ -0,0 +1,1050 @@ +/* +** upb::Decoder (Bytecode Decoder VM) +** +** Bytecode must previously have been generated using the bytecode compiler in +** compile_decoder.c. This decoder then walks through the bytecode op-by-op to +** parse the input. +** +** Decoding is fully resumable; we just keep a pointer to the current bytecode +** instruction and resume from there. A fair amount of the logic here is to +** handle the fact that values can span buffer seams and we have to be able to +** be capable of suspending/resuming from any byte in the stream. This +** sometimes requires keeping a few trailing bytes from the last buffer around +** in the "residual" buffer. +*/ + +#include +#include +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" + +#ifdef UPB_DUMP_BYTECODE +#include +#endif + +#include "upb/port_def.inc" + +#define CHECK_SUSPEND(x) if (!(x)) return upb_pbdecoder_suspend(d); + +/* Error messages that are shared between the bytecode and JIT decoders. */ +const char *kPbDecoderStackOverflow = "Nesting too deep."; +const char *kPbDecoderSubmessageTooLong = + "Submessage end extends past enclosing submessage."; + +/* Error messages shared within this file. */ +static const char *kUnterminatedVarint = "Unterminated varint."; + +/* upb_pbdecoder **************************************************************/ + +static opcode halt = OP_HALT; + +/* A dummy character we can point to when the user passes us a NULL buffer. + * We need this because in C (NULL + 0) and (NULL - NULL) are undefined + * behavior, which would invalidate functions like curbufleft(). */ +static const char dummy_char; + +/* Whether an op consumes any of the input buffer. */ +static bool consumes_input(opcode op) { + switch (op) { + case OP_SETDISPATCH: + case OP_STARTMSG: + case OP_ENDMSG: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_SETBIGGROUPNUM: + case OP_CHECKDELIM: + case OP_CALL: + case OP_RET: + case OP_BRANCH: + return false; + default: + return true; + } +} + +static size_t stacksize(upb_pbdecoder *d, size_t entries) { + UPB_UNUSED(d); + return entries * sizeof(upb_pbdecoder_frame); +} + +static size_t callstacksize(upb_pbdecoder *d, size_t entries) { + UPB_UNUSED(d); + + return entries * sizeof(uint32_t*); +} + + +static bool in_residual_buf(const upb_pbdecoder *d, const char *p); + +/* It's unfortunate that we have to micro-manage the compiler with + * UPB_FORCEINLINE and UPB_NOINLINE, especially since this tuning is necessarily + * specific to one hardware configuration. But empirically on a Core i7, + * performance increases 30-50% with these annotations. Every instance where + * these appear, gcc 4.2.1 made the wrong decision and degraded performance in + * benchmarks. */ + +static void seterr(upb_pbdecoder *d, const char *msg) { + upb_status_seterrmsg(d->status, msg); +} + +void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg) { + seterr(d, msg); +} + + +/* Buffering ******************************************************************/ + +/* We operate on one buffer at a time, which is either the user's buffer passed + * to our "decode" callback or some residual bytes from the previous buffer. */ + +/* How many bytes can be safely read from d->ptr without reading past end-of-buf + * or past the current delimited end. */ +static size_t curbufleft(const upb_pbdecoder *d) { + UPB_ASSERT(d->data_end >= d->ptr); + return d->data_end - d->ptr; +} + +/* How many bytes are available before end-of-buffer. */ +static size_t bufleft(const upb_pbdecoder *d) { + return d->end - d->ptr; +} + +/* Overall stream offset of d->ptr. */ +uint64_t offset(const upb_pbdecoder *d) { + return d->bufstart_ofs + (d->ptr - d->buf); +} + +/* How many bytes are available before the end of this delimited region. */ +size_t delim_remaining(const upb_pbdecoder *d) { + return d->top->end_ofs - offset(d); +} + +/* Advances d->ptr. */ +static void advance(upb_pbdecoder *d, size_t len) { + UPB_ASSERT(curbufleft(d) >= len); + d->ptr += len; +} + +static bool in_buf(const char *p, const char *buf, const char *end) { + return p >= buf && p <= end; +} + +static bool in_residual_buf(const upb_pbdecoder *d, const char *p) { + return in_buf(p, d->residual, d->residual_end); +} + +/* Calculates the delim_end value, which is affected by both the current buffer + * and the parsing stack, so must be called whenever either is updated. */ +static void set_delim_end(upb_pbdecoder *d) { + size_t delim_ofs = d->top->end_ofs - d->bufstart_ofs; + if (delim_ofs <= (size_t)(d->end - d->buf)) { + d->delim_end = d->buf + delim_ofs; + d->data_end = d->delim_end; + } else { + d->data_end = d->end; + d->delim_end = NULL; + } +} + +static void switchtobuf(upb_pbdecoder *d, const char *buf, const char *end) { + d->ptr = buf; + d->buf = buf; + d->end = end; + set_delim_end(d); +} + +static void advancetobuf(upb_pbdecoder *d, const char *buf, size_t len) { + UPB_ASSERT(curbufleft(d) == 0); + d->bufstart_ofs += (d->end - d->buf); + switchtobuf(d, buf, buf + len); +} + +static void checkpoint(upb_pbdecoder *d) { + /* The assertion here is in the interests of efficiency, not correctness. + * We are trying to ensure that we don't checkpoint() more often than + * necessary. */ + UPB_ASSERT(d->checkpoint != d->ptr); + d->checkpoint = d->ptr; +} + +/* Skips "bytes" bytes in the stream, which may be more than available. If we + * skip more bytes than are available, we return a long read count to the caller + * indicating how many bytes can be skipped over before passing actual data + * again. Skipped bytes can pass a NULL buffer and the decoder guarantees they + * won't actually be read. + */ +static int32_t skip(upb_pbdecoder *d, size_t bytes) { + UPB_ASSERT(!in_residual_buf(d, d->ptr) || d->size_param == 0); + UPB_ASSERT(d->skip == 0); + if (bytes > delim_remaining(d)) { + seterr(d, "Skipped value extended beyond enclosing submessage."); + return upb_pbdecoder_suspend(d); + } else if (bufleft(d) >= bytes) { + /* Skipped data is all in current buffer, and more is still available. */ + advance(d, bytes); + d->skip = 0; + return DECODE_OK; + } else { + /* Skipped data extends beyond currently available buffers. */ + d->pc = d->last; + d->skip = bytes - curbufleft(d); + d->bufstart_ofs += (d->end - d->buf); + d->residual_end = d->residual; + switchtobuf(d, d->residual, d->residual_end); + return d->size_param + d->skip; + } +} + + +/* Resumes the decoder from an initial state or from a previous suspend. */ +int32_t upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf, + size_t size, const upb_bufhandle *handle) { + UPB_UNUSED(p); /* Useless; just for the benefit of the JIT. */ + + /* d->skip and d->residual_end could probably elegantly be represented + * as a single variable, to more easily represent this invariant. */ + UPB_ASSERT(!(d->skip && d->residual_end > d->residual)); + + /* We need to remember the original size_param, so that the value we return + * is relative to it, even if we do some skipping first. */ + d->size_param = size; + d->handle = handle; + + /* Have to handle this case specially (ie. not with skip()) because the user + * is allowed to pass a NULL buffer here, which won't allow us to safely + * calculate a d->end or use our normal functions like curbufleft(). */ + if (d->skip && d->skip >= size) { + d->skip -= size; + d->bufstart_ofs += size; + buf = &dummy_char; + size = 0; + + /* We can't just return now, because we might need to execute some ops + * like CHECKDELIM, which could call some callbacks and pop the stack. */ + } + + /* We need to pretend that this was the actual buffer param, since some of the + * calculations assume that d->ptr/d->buf is relative to this. */ + d->buf_param = buf; + + if (!buf) { + /* NULL buf is ok if its entire span is covered by the "skip" above, but + * by this point we know that "skip" doesn't cover the buffer. */ + seterr(d, "Passed NULL buffer over non-skippable region."); + return upb_pbdecoder_suspend(d); + } + + if (d->residual_end > d->residual) { + /* We have residual bytes from the last buffer. */ + UPB_ASSERT(d->ptr == d->residual); + } else { + switchtobuf(d, buf, buf + size); + } + + d->checkpoint = d->ptr; + + /* Handle skips that don't cover the whole buffer (as above). */ + if (d->skip) { + size_t skip_bytes = d->skip; + d->skip = 0; + CHECK_RETURN(skip(d, skip_bytes)); + checkpoint(d); + } + + /* If we're inside an unknown group, continue to parse unknown values. */ + if (d->top->groupnum < 0) { + CHECK_RETURN(upb_pbdecoder_skipunknown(d, -1, 0)); + checkpoint(d); + } + + return DECODE_OK; +} + +/* Suspends the decoder at the last checkpoint, without saving any residual + * bytes. If there are any unconsumed bytes, returns a short byte count. */ +size_t upb_pbdecoder_suspend(upb_pbdecoder *d) { + d->pc = d->last; + if (d->checkpoint == d->residual) { + /* Checkpoint was in residual buf; no user bytes were consumed. */ + d->ptr = d->residual; + return 0; + } else { + size_t ret = d->size_param - (d->end - d->checkpoint); + UPB_ASSERT(!in_residual_buf(d, d->checkpoint)); + UPB_ASSERT(d->buf == d->buf_param || d->buf == &dummy_char); + + d->bufstart_ofs += (d->checkpoint - d->buf); + d->residual_end = d->residual; + switchtobuf(d, d->residual, d->residual_end); + return ret; + } +} + +/* Suspends the decoder at the last checkpoint, and saves any unconsumed + * bytes in our residual buffer. This is necessary if we need more user + * bytes to form a complete value, which might not be contiguous in the + * user's buffers. Always consumes all user bytes. */ +static size_t suspend_save(upb_pbdecoder *d) { + /* We hit end-of-buffer before we could parse a full value. + * Save any unconsumed bytes (if any) to the residual buffer. */ + d->pc = d->last; + + if (d->checkpoint == d->residual) { + /* Checkpoint was in residual buf; append user byte(s) to residual buf. */ + UPB_ASSERT((d->residual_end - d->residual) + d->size_param <= + sizeof(d->residual)); + if (!in_residual_buf(d, d->ptr)) { + d->bufstart_ofs -= (d->residual_end - d->residual); + } + memcpy(d->residual_end, d->buf_param, d->size_param); + d->residual_end += d->size_param; + } else { + /* Checkpoint was in user buf; old residual bytes not needed. */ + size_t save; + UPB_ASSERT(!in_residual_buf(d, d->checkpoint)); + + d->ptr = d->checkpoint; + save = curbufleft(d); + UPB_ASSERT(save <= sizeof(d->residual)); + memcpy(d->residual, d->ptr, save); + d->residual_end = d->residual + save; + d->bufstart_ofs = offset(d); + } + + switchtobuf(d, d->residual, d->residual_end); + return d->size_param; +} + +/* Copies the next "bytes" bytes into "buf" and advances the stream. + * Requires that this many bytes are available in the current buffer. */ +UPB_FORCEINLINE static void consumebytes(upb_pbdecoder *d, void *buf, + size_t bytes) { + UPB_ASSERT(bytes <= curbufleft(d)); + memcpy(buf, d->ptr, bytes); + advance(d, bytes); +} + +/* Slow path for getting the next "bytes" bytes, regardless of whether they are + * available in the current buffer or not. Returns a status code as described + * in decoder.int.h. */ +UPB_NOINLINE static int32_t getbytes_slow(upb_pbdecoder *d, void *buf, + size_t bytes) { + const size_t avail = curbufleft(d); + consumebytes(d, buf, avail); + bytes -= avail; + UPB_ASSERT(bytes > 0); + if (in_residual_buf(d, d->ptr)) { + advancetobuf(d, d->buf_param, d->size_param); + } + if (curbufleft(d) >= bytes) { + consumebytes(d, (char *)buf + avail, bytes); + return DECODE_OK; + } else if (d->data_end == d->delim_end) { + seterr(d, "Submessage ended in the middle of a value or group"); + return upb_pbdecoder_suspend(d); + } else { + return suspend_save(d); + } +} + +/* Gets the next "bytes" bytes, regardless of whether they are available in the + * current buffer or not. Returns a status code as described in decoder.int.h. + */ +UPB_FORCEINLINE static int32_t getbytes(upb_pbdecoder *d, void *buf, + size_t bytes) { + if (curbufleft(d) >= bytes) { + /* Buffer has enough data to satisfy. */ + consumebytes(d, buf, bytes); + return DECODE_OK; + } else { + return getbytes_slow(d, buf, bytes); + } +} + +UPB_NOINLINE static size_t peekbytes_slow(upb_pbdecoder *d, void *buf, + size_t bytes) { + size_t ret = curbufleft(d); + memcpy(buf, d->ptr, ret); + if (in_residual_buf(d, d->ptr)) { + size_t copy = UPB_MIN(bytes - ret, d->size_param); + memcpy((char *)buf + ret, d->buf_param, copy); + ret += copy; + } + return ret; +} + +UPB_FORCEINLINE static size_t peekbytes(upb_pbdecoder *d, void *buf, + size_t bytes) { + if (curbufleft(d) >= bytes) { + memcpy(buf, d->ptr, bytes); + return bytes; + } else { + return peekbytes_slow(d, buf, bytes); + } +} + + +/* Decoding of wire types *****************************************************/ + +/* Slow path for decoding a varint from the current buffer position. + * Returns a status code as described in decoder.int.h. */ +UPB_NOINLINE int32_t upb_pbdecoder_decode_varint_slow(upb_pbdecoder *d, + uint64_t *u64) { + uint8_t byte = 0x80; + int bitpos; + *u64 = 0; + for(bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { + CHECK_RETURN(getbytes(d, &byte, 1)); + *u64 |= (uint64_t)(byte & 0x7F) << bitpos; + } + if(bitpos == 70 && (byte & 0x80)) { + seterr(d, kUnterminatedVarint); + return upb_pbdecoder_suspend(d); + } + return DECODE_OK; +} + +/* Decodes a varint from the current buffer position. + * Returns a status code as described in decoder.int.h. */ +UPB_FORCEINLINE static int32_t decode_varint(upb_pbdecoder *d, uint64_t *u64) { + if (curbufleft(d) > 0 && !(*d->ptr & 0x80)) { + *u64 = *d->ptr; + advance(d, 1); + return DECODE_OK; + } else if (curbufleft(d) >= 10) { + /* Fast case. */ + upb_decoderet r = upb_vdecode_fast(d->ptr); + if (r.p == NULL) { + seterr(d, kUnterminatedVarint); + return upb_pbdecoder_suspend(d); + } + advance(d, r.p - d->ptr); + *u64 = r.val; + return DECODE_OK; + } else { + /* Slow case -- varint spans buffer seam. */ + return upb_pbdecoder_decode_varint_slow(d, u64); + } +} + +/* Decodes a 32-bit varint from the current buffer position. + * Returns a status code as described in decoder.int.h. */ +UPB_FORCEINLINE static int32_t decode_v32(upb_pbdecoder *d, uint32_t *u32) { + uint64_t u64; + int32_t ret = decode_varint(d, &u64); + if (ret >= 0) return ret; + if (u64 > UINT32_MAX) { + seterr(d, "Unterminated 32-bit varint"); + /* TODO(haberman) guarantee that this function return is >= 0 somehow, + * so we know this path will always be treated as error by our caller. + * Right now the size_t -> int32_t can overflow and produce negative values. + */ + *u32 = 0; + return upb_pbdecoder_suspend(d); + } + *u32 = u64; + return DECODE_OK; +} + +/* Decodes a fixed32 from the current buffer position. + * Returns a status code as described in decoder.int.h. + * TODO: proper byte swapping for big-endian machines. */ +UPB_FORCEINLINE static int32_t decode_fixed32(upb_pbdecoder *d, uint32_t *u32) { + return getbytes(d, u32, 4); +} + +/* Decodes a fixed64 from the current buffer position. + * Returns a status code as described in decoder.int.h. + * TODO: proper byte swapping for big-endian machines. */ +UPB_FORCEINLINE static int32_t decode_fixed64(upb_pbdecoder *d, uint64_t *u64) { + return getbytes(d, u64, 8); +} + +/* Non-static versions of the above functions. + * These are called by the JIT for fallback paths. */ +int32_t upb_pbdecoder_decode_f32(upb_pbdecoder *d, uint32_t *u32) { + return decode_fixed32(d, u32); +} + +int32_t upb_pbdecoder_decode_f64(upb_pbdecoder *d, uint64_t *u64) { + return decode_fixed64(d, u64); +} + +static double as_double(uint64_t n) { double d; memcpy(&d, &n, 8); return d; } +static float as_float(uint32_t n) { float f; memcpy(&f, &n, 4); return f; } + +/* Pushes a frame onto the decoder stack. */ +static bool decoder_push(upb_pbdecoder *d, uint64_t end) { + upb_pbdecoder_frame *fr = d->top; + + if (end > fr->end_ofs) { + seterr(d, kPbDecoderSubmessageTooLong); + return false; + } else if (fr == d->limit) { + seterr(d, kPbDecoderStackOverflow); + return false; + } + + fr++; + fr->end_ofs = end; + fr->dispatch = NULL; + fr->groupnum = 0; + d->top = fr; + return true; +} + +static bool pushtagdelim(upb_pbdecoder *d, uint32_t arg) { + /* While we expect to see an "end" tag (either ENDGROUP or a non-sequence + * field number) prior to hitting any enclosing submessage end, pushing our + * existing delim end prevents us from continuing to parse values from a + * corrupt proto that doesn't give us an END tag in time. */ + if (!decoder_push(d, d->top->end_ofs)) + return false; + d->top->groupnum = arg; + return true; +} + +/* Pops a frame from the decoder stack. */ +static void decoder_pop(upb_pbdecoder *d) { d->top--; } + +UPB_NOINLINE int32_t upb_pbdecoder_checktag_slow(upb_pbdecoder *d, + uint64_t expected) { + uint64_t data = 0; + size_t bytes = upb_value_size(expected); + size_t read = peekbytes(d, &data, bytes); + if (read == bytes && data == expected) { + /* Advance past matched bytes. */ + int32_t ok = getbytes(d, &data, read); + UPB_ASSERT(ok < 0); + return DECODE_OK; + } else if (read < bytes && memcmp(&data, &expected, read) == 0) { + return suspend_save(d); + } else { + return DECODE_MISMATCH; + } +} + +int32_t upb_pbdecoder_skipunknown(upb_pbdecoder *d, int32_t fieldnum, + uint8_t wire_type) { + if (fieldnum >= 0) + goto have_tag; + + while (true) { + uint32_t tag; + CHECK_RETURN(decode_v32(d, &tag)); + wire_type = tag & 0x7; + fieldnum = tag >> 3; + +have_tag: + if (fieldnum == 0) { + seterr(d, "Saw invalid field number (0)"); + return upb_pbdecoder_suspend(d); + } + + switch (wire_type) { + case UPB_WIRE_TYPE_32BIT: + CHECK_RETURN(skip(d, 4)); + break; + case UPB_WIRE_TYPE_64BIT: + CHECK_RETURN(skip(d, 8)); + break; + case UPB_WIRE_TYPE_VARINT: { + uint64_t u64; + CHECK_RETURN(decode_varint(d, &u64)); + break; + } + case UPB_WIRE_TYPE_DELIMITED: { + uint32_t len; + CHECK_RETURN(decode_v32(d, &len)); + CHECK_RETURN(skip(d, len)); + break; + } + case UPB_WIRE_TYPE_START_GROUP: + CHECK_SUSPEND(pushtagdelim(d, -fieldnum)); + break; + case UPB_WIRE_TYPE_END_GROUP: + if (fieldnum == -d->top->groupnum) { + decoder_pop(d); + } else if (fieldnum == d->top->groupnum) { + return DECODE_ENDGROUP; + } else { + seterr(d, "Unmatched ENDGROUP tag."); + return upb_pbdecoder_suspend(d); + } + break; + default: + seterr(d, "Invalid wire type"); + return upb_pbdecoder_suspend(d); + } + + if (d->top->groupnum >= 0) { + /* TODO: More code needed for handling unknown groups. */ + upb_sink_putunknown(d->top->sink, d->checkpoint, d->ptr - d->checkpoint); + return DECODE_OK; + } + + /* Unknown group -- continue looping over unknown fields. */ + checkpoint(d); + } +} + +static void goto_endmsg(upb_pbdecoder *d) { + upb_value v; + bool found = upb_inttable_lookup32(d->top->dispatch, DISPATCH_ENDMSG, &v); + UPB_ASSERT(found); + d->pc = d->top->base + upb_value_getuint64(v); +} + +/* Parses a tag and jumps to the corresponding bytecode instruction for this + * field. + * + * If the tag is unknown (or the wire type doesn't match), parses the field as + * unknown. If the tag is a valid ENDGROUP tag, jumps to the bytecode + * instruction for the end of message. */ +static int32_t dispatch(upb_pbdecoder *d) { + upb_inttable *dispatch = d->top->dispatch; + uint32_t tag; + uint8_t wire_type; + uint32_t fieldnum; + upb_value val; + int32_t retval; + + /* Decode tag. */ + CHECK_RETURN(decode_v32(d, &tag)); + wire_type = tag & 0x7; + fieldnum = tag >> 3; + + /* Lookup tag. Because of packed/non-packed compatibility, we have to + * check the wire type against two possibilities. */ + if (fieldnum != DISPATCH_ENDMSG && + upb_inttable_lookup32(dispatch, fieldnum, &val)) { + uint64_t v = upb_value_getuint64(val); + if (wire_type == (v & 0xff)) { + d->pc = d->top->base + (v >> 16); + return DECODE_OK; + } else if (wire_type == ((v >> 8) & 0xff)) { + bool found = + upb_inttable_lookup(dispatch, fieldnum + UPB_MAX_FIELDNUMBER, &val); + UPB_ASSERT(found); + d->pc = d->top->base + upb_value_getuint64(val); + return DECODE_OK; + } + } + + /* We have some unknown fields (or ENDGROUP) to parse. The DISPATCH or TAG + * bytecode that triggered this is preceded by a CHECKDELIM bytecode which + * we need to back up to, so that when we're done skipping unknown data we + * can re-check the delimited end. */ + d->last--; /* Necessary if we get suspended */ + d->pc = d->last; + UPB_ASSERT(getop(*d->last) == OP_CHECKDELIM); + + /* Unknown field or ENDGROUP. */ + retval = upb_pbdecoder_skipunknown(d, fieldnum, wire_type); + + CHECK_RETURN(retval); + + if (retval == DECODE_ENDGROUP) { + goto_endmsg(d); + return DECODE_OK; + } + + return DECODE_OK; +} + +/* Callers know that the stack is more than one deep because the opcodes that + * call this only occur after PUSH operations. */ +upb_pbdecoder_frame *outer_frame(upb_pbdecoder *d) { + UPB_ASSERT(d->top != d->stack); + return d->top - 1; +} + + +/* The main decoding loop *****************************************************/ + +/* The main decoder VM function. Uses traditional bytecode dispatch loop with a + * switch() statement. */ +size_t run_decoder_vm(upb_pbdecoder *d, const mgroup *group, + const upb_bufhandle* handle) { + +#define VMCASE(op, code) \ + case op: { code; if (consumes_input(op)) checkpoint(d); break; } +#define PRIMITIVE_OP(type, wt, name, convfunc, ctype) \ + VMCASE(OP_PARSE_ ## type, { \ + ctype val; \ + CHECK_RETURN(decode_ ## wt(d, &val)); \ + upb_sink_put ## name(d->top->sink, arg, (convfunc)(val)); \ + }) + + while(1) { + int32_t instruction; + opcode op; + uint32_t arg; + int32_t longofs; + + d->last = d->pc; + instruction = *d->pc++; + op = getop(instruction); + arg = instruction >> 8; + longofs = arg; + UPB_ASSERT(d->ptr != d->residual_end); + UPB_UNUSED(group); +#ifdef UPB_DUMP_BYTECODE + fprintf(stderr, "s_ofs=%d buf_ofs=%d data_rem=%d buf_rem=%d delim_rem=%d " + "%x %s (%d)\n", + (int)offset(d), + (int)(d->ptr - d->buf), + (int)(d->data_end - d->ptr), + (int)(d->end - d->ptr), + (int)((d->top->end_ofs - d->bufstart_ofs) - (d->ptr - d->buf)), + (int)(d->pc - 1 - group->bytecode), + upb_pbdecoder_getopname(op), + arg); +#endif + switch (op) { + /* Technically, we are losing data if we see a 32-bit varint that is not + * properly sign-extended. We could detect this and error about the data + * loss, but proto2 does not do this, so we pass. */ + PRIMITIVE_OP(INT32, varint, int32, int32_t, uint64_t) + PRIMITIVE_OP(INT64, varint, int64, int64_t, uint64_t) + PRIMITIVE_OP(UINT32, varint, uint32, uint32_t, uint64_t) + PRIMITIVE_OP(UINT64, varint, uint64, uint64_t, uint64_t) + PRIMITIVE_OP(FIXED32, fixed32, uint32, uint32_t, uint32_t) + PRIMITIVE_OP(FIXED64, fixed64, uint64, uint64_t, uint64_t) + PRIMITIVE_OP(SFIXED32, fixed32, int32, int32_t, uint32_t) + PRIMITIVE_OP(SFIXED64, fixed64, int64, int64_t, uint64_t) + PRIMITIVE_OP(BOOL, varint, bool, bool, uint64_t) + PRIMITIVE_OP(DOUBLE, fixed64, double, as_double, uint64_t) + PRIMITIVE_OP(FLOAT, fixed32, float, as_float, uint32_t) + PRIMITIVE_OP(SINT32, varint, int32, upb_zzdec_32, uint64_t) + PRIMITIVE_OP(SINT64, varint, int64, upb_zzdec_64, uint64_t) + + VMCASE(OP_SETDISPATCH, + d->top->base = d->pc - 1; + memcpy(&d->top->dispatch, d->pc, sizeof(void*)); + d->pc += sizeof(void*) / sizeof(uint32_t); + ) + VMCASE(OP_STARTMSG, + CHECK_SUSPEND(upb_sink_startmsg(d->top->sink)); + ) + VMCASE(OP_ENDMSG, + CHECK_SUSPEND(upb_sink_endmsg(d->top->sink, d->status)); + ) + VMCASE(OP_STARTSEQ, + upb_pbdecoder_frame *outer = outer_frame(d); + CHECK_SUSPEND(upb_sink_startseq(outer->sink, arg, &d->top->sink)); + ) + VMCASE(OP_ENDSEQ, + CHECK_SUSPEND(upb_sink_endseq(d->top->sink, arg)); + ) + VMCASE(OP_STARTSUBMSG, + upb_pbdecoder_frame *outer = outer_frame(d); + CHECK_SUSPEND(upb_sink_startsubmsg(outer->sink, arg, &d->top->sink)); + ) + VMCASE(OP_ENDSUBMSG, + CHECK_SUSPEND(upb_sink_endsubmsg(d->top->sink, arg)); + ) + VMCASE(OP_STARTSTR, + uint32_t len = delim_remaining(d); + upb_pbdecoder_frame *outer = outer_frame(d); + CHECK_SUSPEND(upb_sink_startstr(outer->sink, arg, len, &d->top->sink)); + if (len == 0) { + d->pc++; /* Skip OP_STRING. */ + } + ) + VMCASE(OP_STRING, + uint32_t len = curbufleft(d); + size_t n = upb_sink_putstring(d->top->sink, arg, d->ptr, len, handle); + if (n > len) { + if (n > delim_remaining(d)) { + seterr(d, "Tried to skip past end of string."); + return upb_pbdecoder_suspend(d); + } else { + int32_t ret = skip(d, n); + /* This shouldn't return DECODE_OK, because n > len. */ + UPB_ASSERT(ret >= 0); + return ret; + } + } + advance(d, n); + if (n < len || d->delim_end == NULL) { + /* We aren't finished with this string yet. */ + d->pc--; /* Repeat OP_STRING. */ + if (n > 0) checkpoint(d); + return upb_pbdecoder_suspend(d); + } + ) + VMCASE(OP_ENDSTR, + CHECK_SUSPEND(upb_sink_endstr(d->top->sink, arg)); + ) + VMCASE(OP_PUSHTAGDELIM, + CHECK_SUSPEND(pushtagdelim(d, arg)); + ) + VMCASE(OP_SETBIGGROUPNUM, + d->top->groupnum = *d->pc++; + ) + VMCASE(OP_POP, + UPB_ASSERT(d->top > d->stack); + decoder_pop(d); + ) + VMCASE(OP_PUSHLENDELIM, + uint32_t len; + CHECK_RETURN(decode_v32(d, &len)); + CHECK_SUSPEND(decoder_push(d, offset(d) + len)); + set_delim_end(d); + ) + VMCASE(OP_SETDELIM, + set_delim_end(d); + ) + VMCASE(OP_CHECKDELIM, + /* We are guaranteed of this assert because we never allow ourselves to + * consume bytes beyond data_end, which covers delim_end when non-NULL. + */ + UPB_ASSERT(!(d->delim_end && d->ptr > d->delim_end)); + if (d->ptr == d->delim_end) + d->pc += longofs; + ) + VMCASE(OP_CALL, + d->callstack[d->call_len++] = d->pc; + d->pc += longofs; + ) + VMCASE(OP_RET, + UPB_ASSERT(d->call_len > 0); + d->pc = d->callstack[--d->call_len]; + ) + VMCASE(OP_BRANCH, + d->pc += longofs; + ) + VMCASE(OP_TAG1, + uint8_t expected; + CHECK_SUSPEND(curbufleft(d) > 0); + expected = (arg >> 8) & 0xff; + if (*d->ptr == expected) { + advance(d, 1); + } else { + int8_t shortofs; + badtag: + shortofs = arg; + if (shortofs == LABEL_DISPATCH) { + CHECK_RETURN(dispatch(d)); + } else { + d->pc += shortofs; + break; /* Avoid checkpoint(). */ + } + } + ) + VMCASE(OP_TAG2, + uint16_t expected; + CHECK_SUSPEND(curbufleft(d) > 0); + expected = (arg >> 8) & 0xffff; + if (curbufleft(d) >= 2) { + uint16_t actual; + memcpy(&actual, d->ptr, 2); + if (expected == actual) { + advance(d, 2); + } else { + goto badtag; + } + } else { + int32_t result = upb_pbdecoder_checktag_slow(d, expected); + if (result == DECODE_MISMATCH) goto badtag; + if (result >= 0) return result; + } + ) + VMCASE(OP_TAGN, { + uint64_t expected; + int32_t result; + memcpy(&expected, d->pc, 8); + d->pc += 2; + result = upb_pbdecoder_checktag_slow(d, expected); + if (result == DECODE_MISMATCH) goto badtag; + if (result >= 0) return result; + }) + VMCASE(OP_DISPATCH, { + CHECK_RETURN(dispatch(d)); + }) + VMCASE(OP_HALT, { + return d->size_param; + }) + } + } +} + + +/* BytesHandler handlers ******************************************************/ + +void *upb_pbdecoder_startbc(void *closure, const void *pc, size_t size_hint) { + upb_pbdecoder *d = closure; + UPB_UNUSED(size_hint); + d->top->end_ofs = UINT64_MAX; + d->bufstart_ofs = 0; + d->call_len = 1; + d->callstack[0] = &halt; + d->pc = pc; + d->skip = 0; + return d; +} + +bool upb_pbdecoder_end(void *closure, const void *handler_data) { + upb_pbdecoder *d = closure; + const upb_pbdecodermethod *method = handler_data; + uint64_t end; + char dummy; + + if (d->residual_end > d->residual) { + seterr(d, "Unexpected EOF: decoder still has buffered unparsed data"); + return false; + } + + if (d->skip) { + seterr(d, "Unexpected EOF inside skipped data"); + return false; + } + + if (d->top->end_ofs != UINT64_MAX) { + seterr(d, "Unexpected EOF inside delimited string"); + return false; + } + + /* The user's end() call indicates that the message ends here. */ + end = offset(d); + d->top->end_ofs = end; + + { + const uint32_t *p = d->pc; + d->stack->end_ofs = end; + /* Check the previous bytecode, but guard against beginning. */ + if (p != method->code_base.ptr) p--; + if (getop(*p) == OP_CHECKDELIM) { + /* Rewind from OP_TAG* to OP_CHECKDELIM. */ + UPB_ASSERT(getop(*d->pc) == OP_TAG1 || + getop(*d->pc) == OP_TAG2 || + getop(*d->pc) == OP_TAGN || + getop(*d->pc) == OP_DISPATCH); + d->pc = p; + } + upb_pbdecoder_decode(closure, handler_data, &dummy, 0, NULL); + } + + if (d->call_len != 0) { + seterr(d, "Unexpected EOF inside submessage or group"); + return false; + } + + return true; +} + +size_t upb_pbdecoder_decode(void *decoder, const void *group, const char *buf, + size_t size, const upb_bufhandle *handle) { + int32_t result = upb_pbdecoder_resume(decoder, NULL, buf, size, handle); + + if (result == DECODE_ENDGROUP) goto_endmsg(decoder); + CHECK_RETURN(result); + + return run_decoder_vm(decoder, group, handle); +} + + +/* Public API *****************************************************************/ + +void upb_pbdecoder_reset(upb_pbdecoder *d) { + d->top = d->stack; + d->top->groupnum = 0; + d->ptr = d->residual; + d->buf = d->residual; + d->end = d->residual; + d->residual_end = d->residual; +} + +upb_pbdecoder *upb_pbdecoder_create(upb_arena *a, const upb_pbdecodermethod *m, + upb_sink sink, upb_status *status) { + const size_t default_max_nesting = 64; +#ifndef NDEBUG + size_t size_before = upb_arena_bytesallocated(a); +#endif + + upb_pbdecoder *d = upb_arena_malloc(a, sizeof(upb_pbdecoder)); + if (!d) return NULL; + + d->method_ = m; + d->callstack = upb_arena_malloc(a, callstacksize(d, default_max_nesting)); + d->stack = upb_arena_malloc(a, stacksize(d, default_max_nesting)); + if (!d->stack || !d->callstack) { + return NULL; + } + + d->arena = a; + d->limit = d->stack + default_max_nesting - 1; + d->stack_size = default_max_nesting; + d->status = status; + + upb_pbdecoder_reset(d); + upb_bytessink_reset(&d->input_, &m->input_handler_, d); + + if (d->method_->dest_handlers_) { + if (sink.handlers != d->method_->dest_handlers_) + return NULL; + } + d->top->sink = sink; + + /* If this fails, increase the value in decoder.h. */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(a) - size_before <= + UPB_PB_DECODER_SIZE); + return d; +} + +uint64_t upb_pbdecoder_bytesparsed(const upb_pbdecoder *d) { + return offset(d); +} + +const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d) { + return d->method_; +} + +upb_bytessink upb_pbdecoder_input(upb_pbdecoder *d) { + return d->input_; +} + +size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d) { + return d->stack_size; +} + +bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max) { + UPB_ASSERT(d->top >= d->stack); + + if (max < (size_t)(d->top - d->stack)) { + /* Can't set a limit smaller than what we are currently at. */ + return false; + } + + if (max > d->stack_size) { + /* Need to reallocate stack and callstack to accommodate. */ + size_t old_size = stacksize(d, d->stack_size); + size_t new_size = stacksize(d, max); + void *p = upb_arena_realloc(d->arena, d->stack, old_size, new_size); + if (!p) { + return false; + } + d->stack = p; + + old_size = callstacksize(d, d->stack_size); + new_size = callstacksize(d, max); + p = upb_arena_realloc(d->arena, d->callstack, old_size, new_size); + if (!p) { + return false; + } + d->callstack = p; + + d->stack_size = max; + } + + d->limit = d->stack + max - 1; + return true; +} diff --git a/upb/pb/decoder.h b/upb/pb/decoder.h new file mode 100644 index 00000000000..709db49e96a --- /dev/null +++ b/upb/pb/decoder.h @@ -0,0 +1,240 @@ +/* +** upb::pb::Decoder +** +** A high performance, streaming, resumable decoder for the binary protobuf +** format. +** +** This interface works the same regardless of what decoder backend is being +** used. A client of this class does not need to know whether decoding is using +** a JITted decoder (DynASM, LLVM, etc) or an interpreted decoder. By default, +** it will always use the fastest available decoder. However, you can call +** set_allow_jit(false) to disable any JIT decoder that might be available. +** This is primarily useful for testing purposes. +*/ + +#ifndef UPB_DECODER_H_ +#define UPB_DECODER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace pb { +class CodeCache; +class DecoderPtr; +class DecoderMethodPtr; +class DecoderMethodOptions; +} /* namespace pb */ +} /* namespace upb */ +#endif + +/* The maximum number of bytes we are required to buffer internally between + * calls to the decoder. The value is 14: a 5 byte unknown tag plus ten-byte + * varint, less one because we are buffering an incomplete value. + * + * Should only be used by unit tests. */ +#define UPB_DECODER_MAX_RESIDUAL_BYTES 14 + +/* upb_pbdecodermethod ********************************************************/ + +struct upb_pbdecodermethod; +typedef struct upb_pbdecodermethod upb_pbdecodermethod; + +#ifdef __cplusplus +extern "C" { +#endif + +const upb_handlers *upb_pbdecodermethod_desthandlers( + const upb_pbdecodermethod *m); +const upb_byteshandler *upb_pbdecodermethod_inputhandler( + const upb_pbdecodermethod *m); +bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m); + +#ifdef __cplusplus +} /* extern "C" */ + +/* Represents the code to parse a protobuf according to a destination + * Handlers. */ +class upb::pb::DecoderMethodPtr { + public: + DecoderMethodPtr() : ptr_(nullptr) {} + DecoderMethodPtr(const upb_pbdecodermethod* ptr) : ptr_(ptr) {} + + const upb_pbdecodermethod* ptr() { return ptr_; } + + /* The destination handlers that are statically bound to this method. + * This method is only capable of outputting to a sink that uses these + * handlers. */ + const Handlers *dest_handlers() const { + return upb_pbdecodermethod_desthandlers(ptr_); + } + + /* The input handlers for this decoder method. */ + const BytesHandler* input_handler() const { + return upb_pbdecodermethod_inputhandler(ptr_); + } + + /* Whether this method is native. */ + bool is_native() const { + return upb_pbdecodermethod_isnative(ptr_); + } + + private: + const upb_pbdecodermethod* ptr_; +}; + +#endif + +/* upb_pbdecoder **************************************************************/ + +/* Preallocation hint: decoder won't allocate more bytes than this when first + * constructed. This hint may be an overestimate for some build configurations. + * But if the decoder library is upgraded without recompiling the application, + * it may be an underestimate. */ +#define UPB_PB_DECODER_SIZE 4416 + +struct upb_pbdecoder; +typedef struct upb_pbdecoder upb_pbdecoder; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_pbdecoder *upb_pbdecoder_create(upb_arena *arena, + const upb_pbdecodermethod *method, + upb_sink output, upb_status *status); +const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d); +upb_bytessink upb_pbdecoder_input(upb_pbdecoder *d); +uint64_t upb_pbdecoder_bytesparsed(const upb_pbdecoder *d); +size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d); +bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max); +void upb_pbdecoder_reset(upb_pbdecoder *d); + +#ifdef __cplusplus +} /* extern "C" */ + +/* A Decoder receives binary protobuf data on its input sink and pushes the + * decoded data to its output sink. */ +class upb::pb::DecoderPtr { + public: + DecoderPtr() : ptr_(nullptr) {} + DecoderPtr(upb_pbdecoder* ptr) : ptr_(ptr) {} + + upb_pbdecoder* ptr() { return ptr_; } + + /* Constructs a decoder instance for the given method, which must outlive this + * decoder. Any errors during parsing will be set on the given status, which + * must also outlive this decoder. + * + * The sink must match the given method. */ + static DecoderPtr Create(Arena *arena, DecoderMethodPtr method, + upb::Sink output, Status *status) { + return DecoderPtr(upb_pbdecoder_create(arena->ptr(), method.ptr(), + output.sink(), status->ptr())); + } + + /* Returns the DecoderMethod this decoder is parsing from. */ + const DecoderMethodPtr method() const { + return DecoderMethodPtr(upb_pbdecoder_method(ptr_)); + } + + /* The sink on which this decoder receives input. */ + BytesSink input() { return BytesSink(upb_pbdecoder_input(ptr())); } + + /* Returns number of bytes successfully parsed. + * + * This can be useful for determining the stream position where an error + * occurred. + * + * This value may not be up-to-date when called from inside a parsing + * callback. */ + uint64_t BytesParsed() { return upb_pbdecoder_bytesparsed(ptr()); } + + /* Gets/sets the parsing nexting limit. If the total number of nested + * submessages and repeated fields hits this limit, parsing will fail. This + * is a resource limit that controls the amount of memory used by the parsing + * stack. + * + * Setting the limit will fail if the parser is currently suspended at a depth + * greater than this, or if memory allocation of the stack fails. */ + size_t max_nesting() { return upb_pbdecoder_maxnesting(ptr()); } + bool set_max_nesting(size_t max) { return upb_pbdecoder_maxnesting(ptr()); } + + void Reset() { upb_pbdecoder_reset(ptr()); } + + static const size_t kSize = UPB_PB_DECODER_SIZE; + + private: + upb_pbdecoder *ptr_; +}; + +#endif /* __cplusplus */ + +/* upb_pbcodecache ************************************************************/ + +/* Lazily builds and caches decoder methods that will push data to the given + * handlers. The destination handlercache must outlive this object. */ + +struct upb_pbcodecache; +typedef struct upb_pbcodecache upb_pbcodecache; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_pbcodecache *upb_pbcodecache_new(upb_handlercache *dest); +void upb_pbcodecache_free(upb_pbcodecache *c); +bool upb_pbcodecache_allowjit(const upb_pbcodecache *c); +void upb_pbcodecache_setallowjit(upb_pbcodecache *c, bool allow); +void upb_pbcodecache_setlazy(upb_pbcodecache *c, bool lazy); +const upb_pbdecodermethod *upb_pbcodecache_get(upb_pbcodecache *c, + const upb_msgdef *md); + +#ifdef __cplusplus +} /* extern "C" */ + +/* A class for caching protobuf processing code, whether bytecode for the + * interpreted decoder or machine code for the JIT. + * + * This class is not thread-safe. */ +class upb::pb::CodeCache { + public: + CodeCache(upb::HandlerCache *dest) + : ptr_(upb_pbcodecache_new(dest->ptr()), upb_pbcodecache_free) {} + CodeCache(CodeCache&&) = default; + CodeCache& operator=(CodeCache&&) = default; + + upb_pbcodecache* ptr() { return ptr_.get(); } + const upb_pbcodecache* ptr() const { return ptr_.get(); } + + /* Whether the cache is allowed to generate machine code. Defaults to true. + * There is no real reason to turn it off except for testing or if you are + * having a specific problem with the JIT. + * + * Note that allow_jit = true does not *guarantee* that the code will be JIT + * compiled. If this platform is not supported or the JIT was not compiled + * in, the code may still be interpreted. */ + bool allow_jit() const { return upb_pbcodecache_allowjit(ptr()); } + + /* This may only be called when the object is first constructed, and prior to + * any code generation. */ + void set_allow_jit(bool allow) { upb_pbcodecache_setallowjit(ptr(), allow); } + + /* Should the decoder push submessages to lazy handlers for fields that have + * them? The caller should set this iff the lazy handlers expect data that is + * in protobuf binary format and the caller wishes to lazy parse it. */ + void set_lazy(bool lazy) { upb_pbcodecache_setlazy(ptr(), lazy); } + + /* Returns a DecoderMethod that can push data to the given handlers. + * If a suitable method already exists, it will be returned from the cache. */ + const DecoderMethodPtr Get(MessageDefPtr md) { + return DecoderMethodPtr(upb_pbcodecache_get(ptr(), md.ptr())); + } + + private: + std::unique_ptr ptr_; +}; + +#endif /* __cplusplus */ + +#endif /* UPB_DECODER_H_ */ diff --git a/upb/pb/decoder.int.h b/upb/pb/decoder.int.h new file mode 100644 index 00000000000..9d5f5839bc8 --- /dev/null +++ b/upb/pb/decoder.int.h @@ -0,0 +1,288 @@ +/* +** Internal-only definitions for the decoder. +*/ + +#ifndef UPB_DECODER_INT_H_ +#define UPB_DECODER_INT_H_ + +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/pb/decoder.h" +#include "upb/sink.h" +#include "upb/table.int.h" + +#include "upb/port_def.inc" + +/* Opcode definitions. The canonical meaning of each opcode is its + * implementation in the interpreter (the JIT is written to match this). + * + * All instructions have the opcode in the low byte. + * Instruction format for most instructions is: + * + * +-------------------+--------+ + * | arg (24) | op (8) | + * +-------------------+--------+ + * + * Exceptions are indicated below. A few opcodes are multi-word. */ +typedef enum { + /* Opcodes 1-8, 13, 15-18 parse their respective descriptor types. + * Arg for all of these is the upb selector for this field. */ +#define T(type) OP_PARSE_ ## type = UPB_DESCRIPTOR_TYPE_ ## type + T(DOUBLE), T(FLOAT), T(INT64), T(UINT64), T(INT32), T(FIXED64), T(FIXED32), + T(BOOL), T(UINT32), T(SFIXED32), T(SFIXED64), T(SINT32), T(SINT64), +#undef T + OP_STARTMSG = 9, /* No arg. */ + OP_ENDMSG = 10, /* No arg. */ + OP_STARTSEQ = 11, + OP_ENDSEQ = 12, + OP_STARTSUBMSG = 14, + OP_ENDSUBMSG = 19, + OP_STARTSTR = 20, + OP_STRING = 21, + OP_ENDSTR = 22, + + OP_PUSHTAGDELIM = 23, /* No arg. */ + OP_PUSHLENDELIM = 24, /* No arg. */ + OP_POP = 25, /* No arg. */ + OP_SETDELIM = 26, /* No arg. */ + OP_SETBIGGROUPNUM = 27, /* two words: + * | unused (24) | opc (8) | + * | groupnum (32) | */ + OP_CHECKDELIM = 28, + OP_CALL = 29, + OP_RET = 30, + OP_BRANCH = 31, + + /* Different opcodes depending on how many bytes expected. */ + OP_TAG1 = 32, /* | match tag (16) | jump target (8) | opc (8) | */ + OP_TAG2 = 33, /* | match tag (16) | jump target (8) | opc (8) | */ + OP_TAGN = 34, /* three words: */ + /* | unused (16) | jump target(8) | opc (8) | */ + /* | match tag 1 (32) | */ + /* | match tag 2 (32) | */ + + OP_SETDISPATCH = 35, /* N words: */ + /* | unused (24) | opc | */ + /* | upb_inttable* (32 or 64) | */ + + OP_DISPATCH = 36, /* No arg. */ + + OP_HALT = 37 /* No arg. */ +} opcode; + +#define OP_MAX OP_HALT + +UPB_INLINE opcode getop(uint32_t instr) { return (opcode)(instr & 0xff); } + +struct upb_pbcodecache { + upb_arena *arena; + upb_handlercache *dest; + bool allow_jit; + bool lazy; + + /* Map of upb_msgdef -> mgroup. */ + upb_inttable groups; +}; + +/* Method group; represents a set of decoder methods that had their code + * emitted together. Immutable once created. */ +typedef struct { + /* Maps upb_msgdef/upb_handlers -> upb_pbdecodermethod. Owned by us. + * + * Ideally this would be on pbcodecache (if we were actually caching code). + * Right now we don't actually cache anything, which is wasteful. */ + upb_inttable methods; + + /* The bytecode for our methods, if any exists. Owned by us. */ + uint32_t *bytecode; + uint32_t *bytecode_end; +} mgroup; + +/* The maximum that any submessages can be nested. Matches proto2's limit. + * This specifies the size of the decoder's statically-sized array and therefore + * setting it high will cause the upb::pb::Decoder object to be larger. + * + * If necessary we can add a runtime-settable property to Decoder that allow + * this to be larger than the compile-time setting, but this would add + * complexity, particularly since we would have to decide how/if to give users + * the ability to set a custom memory allocation function. */ +#define UPB_DECODER_MAX_NESTING 64 + +/* Internal-only struct used by the decoder. */ +typedef struct { + /* Space optimization note: we store two pointers here that the JIT + * doesn't need at all; the upb_handlers* inside the sink and + * the dispatch table pointer. We can optimze so that the JIT uses + * smaller stack frames than the interpreter. The only thing we need + * to guarantee is that the fallback routines can find end_ofs. */ + upb_sink sink; + + /* The absolute stream offset of the end-of-frame delimiter. + * Non-delimited frames (groups and non-packed repeated fields) reuse the + * delimiter of their parent, even though the frame may not end there. + * + * NOTE: the JIT stores a slightly different value here for non-top frames. + * It stores the value relative to the end of the enclosed message. But the + * top frame is still stored the same way, which is important for ensuring + * that calls from the JIT into C work correctly. */ + uint64_t end_ofs; + const uint32_t *base; + + /* 0 indicates a length-delimited field. + * A positive number indicates a known group. + * A negative number indicates an unknown group. */ + int32_t groupnum; + upb_inttable *dispatch; /* Not used by the JIT. */ +} upb_pbdecoder_frame; + +struct upb_pbdecodermethod { + /* While compiling, the base is relative in "ofs", after compiling it is + * absolute in "ptr". */ + union { + uint32_t ofs; /* PC offset of method. */ + void *ptr; /* Pointer to bytecode or machine code for this method. */ + } code_base; + + /* The decoder method group to which this method belongs. */ + const mgroup *group; + + /* Whether this method is native code or bytecode. */ + bool is_native_; + + /* The handler one calls to invoke this method. */ + upb_byteshandler input_handler_; + + /* The destination handlers this method is bound to. We own a ref. */ + const upb_handlers *dest_handlers_; + + /* Dispatch table -- used by both bytecode decoder and JIT when encountering a + * field number that wasn't the one we were expecting to see. See + * decoder.int.h for the layout of this table. */ + upb_inttable dispatch; +}; + +struct upb_pbdecoder { + upb_arena *arena; + + /* Our input sink. */ + upb_bytessink input_; + + /* The decoder method we are parsing with (owned). */ + const upb_pbdecodermethod *method_; + + size_t call_len; + const uint32_t *pc, *last; + + /* Current input buffer and its stream offset. */ + const char *buf, *ptr, *end, *checkpoint; + + /* End of the delimited region, relative to ptr, NULL if not in this buf. */ + const char *delim_end; + + /* End of the delimited region, relative to ptr, end if not in this buf. */ + const char *data_end; + + /* Overall stream offset of "buf." */ + uint64_t bufstart_ofs; + + /* Buffer for residual bytes not parsed from the previous buffer. */ + char residual[UPB_DECODER_MAX_RESIDUAL_BYTES]; + char *residual_end; + + /* Bytes of data that should be discarded from the input beore we start + * parsing again. We set this when we internally determine that we can + * safely skip the next N bytes, but this region extends past the current + * user buffer. */ + size_t skip; + + /* Stores the user buffer passed to our decode function. */ + const char *buf_param; + size_t size_param; + const upb_bufhandle *handle; + + /* Our internal stack. */ + upb_pbdecoder_frame *stack, *top, *limit; + const uint32_t **callstack; + size_t stack_size; + + upb_status *status; +}; + +/* Decoder entry points; used as handlers. */ +void *upb_pbdecoder_startbc(void *closure, const void *pc, size_t size_hint); +size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf, + size_t size, const upb_bufhandle *handle); +bool upb_pbdecoder_end(void *closure, const void *handler_data); + +/* Decoder-internal functions that the JIT calls to handle fallback paths. */ +int32_t upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf, + size_t size, const upb_bufhandle *handle); +size_t upb_pbdecoder_suspend(upb_pbdecoder *d); +int32_t upb_pbdecoder_skipunknown(upb_pbdecoder *d, int32_t fieldnum, + uint8_t wire_type); +int32_t upb_pbdecoder_checktag_slow(upb_pbdecoder *d, uint64_t expected); +int32_t upb_pbdecoder_decode_varint_slow(upb_pbdecoder *d, uint64_t *u64); +int32_t upb_pbdecoder_decode_f32(upb_pbdecoder *d, uint32_t *u32); +int32_t upb_pbdecoder_decode_f64(upb_pbdecoder *d, uint64_t *u64); +void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg); + +/* Error messages that are shared between the bytecode and JIT decoders. */ +extern const char *kPbDecoderStackOverflow; +extern const char *kPbDecoderSubmessageTooLong; + +/* Access to decoderplan members needed by the decoder. */ +const char *upb_pbdecoder_getopname(unsigned int op); + +/* A special label that means "do field dispatch for this message and branch to + * wherever that takes you." */ +#define LABEL_DISPATCH 0 + +/* A special slot in the dispatch table that stores the epilogue (ENDMSG and/or + * RET) for branching to when we find an appropriate ENDGROUP tag. */ +#define DISPATCH_ENDMSG 0 + +/* It's important to use this invalid wire type instead of 0 (which is a valid + * wire type). */ +#define NO_WIRE_TYPE 0xff + +/* The dispatch table layout is: + * [field number] -> [ 48-bit offset ][ 8-bit wt2 ][ 8-bit wt1 ] + * + * If wt1 matches, jump to the 48-bit offset. If wt2 matches, lookup + * (UPB_MAX_FIELDNUMBER + fieldnum) and jump there. + * + * We need two wire types because of packed/non-packed compatibility. A + * primitive repeated field can use either wire type and be valid. While we + * could key the table on fieldnum+wiretype, the table would be 8x sparser. + * + * Storing two wire types in the primary value allows us to quickly rule out + * the second wire type without needing to do a separate lookup (this case is + * less common than an unknown field). */ +UPB_INLINE uint64_t upb_pbdecoder_packdispatch(uint64_t ofs, uint8_t wt1, + uint8_t wt2) { + return (ofs << 16) | (wt2 << 8) | wt1; +} + +UPB_INLINE void upb_pbdecoder_unpackdispatch(uint64_t dispatch, uint64_t *ofs, + uint8_t *wt1, uint8_t *wt2) { + *wt1 = (uint8_t)dispatch; + *wt2 = (uint8_t)(dispatch >> 8); + *ofs = dispatch >> 16; +} + +/* All of the functions in decoder.c that return int32_t return values according + * to the following scheme: + * 1. negative values indicate a return code from the following list. + * 2. positive values indicate that error or end of buffer was hit, and + * that the decode function should immediately return the given value + * (the decoder state has already been suspended and is ready to be + * resumed). */ +#define DECODE_OK -1 +#define DECODE_MISMATCH -2 /* Used only from checktag_slow(). */ +#define DECODE_ENDGROUP -3 /* Used only from checkunknown(). */ + +#define CHECK_RETURN(x) { int32_t ret = x; if (ret >= 0) return ret; } + +#include "upb/port_undef.inc" + +#endif /* UPB_DECODER_INT_H_ */ diff --git a/upb/pb/encoder.c b/upb/pb/encoder.c new file mode 100644 index 00000000000..d108a643d30 --- /dev/null +++ b/upb/pb/encoder.c @@ -0,0 +1,570 @@ +/* +** upb::Encoder +** +** Since we are implementing pure handlers (ie. without any out-of-band access +** to pre-computed lengths), we have to buffer all submessages before we can +** emit even their first byte. +** +** Not knowing the size of submessages also means we can't write a perfect +** zero-copy implementation, even with buffering. Lengths are stored as +** varints, which means that we don't know how many bytes to reserve for the +** length until we know what the length is. +** +** This leaves us with three main choices: +** +** 1. buffer all submessage data in a temporary buffer, then copy it exactly +** once into the output buffer. +** +** 2. attempt to buffer data directly into the output buffer, estimating how +** many bytes each length will take. When our guesses are wrong, use +** memmove() to grow or shrink the allotted space. +** +** 3. buffer directly into the output buffer, allocating a max length +** ahead-of-time for each submessage length. If we overallocated, we waste +** space, but no memcpy() or memmove() is required. This approach requires +** defining a maximum size for submessages and rejecting submessages that +** exceed that size. +** +** (2) and (3) have the potential to have better performance, but they are more +** complicated and subtle to implement: +** +** (3) requires making an arbitrary choice of the maximum message size; it +** wastes space when submessages are shorter than this and fails +** completely when they are longer. This makes it more finicky and +** requires configuration based on the input. It also makes it impossible +** to perfectly match the output of reference encoders that always use the +** optimal amount of space for each length. +** +** (2) requires guessing the the size upfront, and if multiple lengths are +** guessed wrong the minimum required number of memmove() operations may +** be complicated to compute correctly. Implemented properly, it may have +** a useful amortized or average cost, but more investigation is required +** to determine this and what the optimal algorithm is to achieve it. +** +** (1) makes you always pay for exactly one copy, but its implementation is +** the simplest and its performance is predictable. +** +** So for now, we implement (1) only. If we wish to optimize later, we should +** be able to do it without affecting users. +** +** The strategy is to buffer the segments of data that do *not* depend on +** unknown lengths in one buffer, and keep a separate buffer of segment pointers +** and lengths. When the top-level submessage ends, we can go beginning to end, +** alternating the writing of lengths with memcpy() of the rest of the data. +** At the top level though, no buffering is required. +*/ + +#include "upb/pb/encoder.h" +#include "upb/pb/varint.int.h" + +#include "upb/port_def.inc" + +/* The output buffer is divided into segments; a segment is a string of data + * that is "ready to go" -- it does not need any varint lengths inserted into + * the middle. The seams between segments are where varints will be inserted + * once they are known. + * + * We also use the concept of a "run", which is a range of encoded bytes that + * occur at a single submessage level. Every segment contains one or more runs. + * + * A segment can span messages. Consider: + * + * .--Submessage lengths---------. + * | | | + * | V V + * V | |--------------- | |----------------- + * Submessages: | |----------------------------------------------- + * Top-level msg: ------------------------------------------------------------ + * + * Segments: ----- ------------------- ----------------- + * Runs: *---- *--------------*--- *---------------- + * (* marks the start) + * + * Note that the top-level menssage is not in any segment because it does not + * have any length preceding it. + * + * A segment is only interrupted when another length needs to be inserted. So + * observe how the second segment spans both the inner submessage and part of + * the next enclosing message. */ +typedef struct { + uint32_t msglen; /* The length to varint-encode before this segment. */ + uint32_t seglen; /* Length of the segment. */ +} upb_pb_encoder_segment; + +struct upb_pb_encoder { + upb_arena *arena; + + /* Our input and output. */ + upb_sink input_; + upb_bytessink output_; + + /* The "subclosure" -- used as the inner closure as part of the bytessink + * protocol. */ + void *subc; + + /* The output buffer and limit, and our current write position. "buf" + * initially points to "initbuf", but is dynamically allocated if we need to + * grow beyond the initial size. */ + char *buf, *ptr, *limit; + + /* The beginning of the current run, or undefined if we are at the top + * level. */ + char *runbegin; + + /* The list of segments we are accumulating. */ + upb_pb_encoder_segment *segbuf, *segptr, *seglimit; + + /* The stack of enclosing submessages. Each entry in the stack points to the + * segment where this submessage's length is being accumulated. */ + int *stack, *top, *stacklimit; + + /* Depth of startmsg/endmsg calls. */ + int depth; +}; + +/* low-level buffering ********************************************************/ + +/* Low-level functions for interacting with the output buffer. */ + +/* TODO(haberman): handle pushback */ +static void putbuf(upb_pb_encoder *e, const char *buf, size_t len) { + size_t n = upb_bytessink_putbuf(e->output_, e->subc, buf, len, NULL); + UPB_ASSERT(n == len); +} + +static upb_pb_encoder_segment *top(upb_pb_encoder *e) { + return &e->segbuf[*e->top]; +} + +/* Call to ensure that at least "bytes" bytes are available for writing at + * e->ptr. Returns false if the bytes could not be allocated. */ +static bool reserve(upb_pb_encoder *e, size_t bytes) { + if ((size_t)(e->limit - e->ptr) < bytes) { + /* Grow buffer. */ + char *new_buf; + size_t needed = bytes + (e->ptr - e->buf); + size_t old_size = e->limit - e->buf; + + size_t new_size = old_size; + + while (new_size < needed) { + new_size *= 2; + } + + new_buf = upb_arena_realloc(e->arena, e->buf, old_size, new_size); + + if (new_buf == NULL) { + return false; + } + + e->ptr = new_buf + (e->ptr - e->buf); + e->runbegin = new_buf + (e->runbegin - e->buf); + e->limit = new_buf + new_size; + e->buf = new_buf; + } + + return true; +} + +/* Call when "bytes" bytes have been writte at e->ptr. The caller *must* have + * previously called reserve() with at least this many bytes. */ +static void encoder_advance(upb_pb_encoder *e, size_t bytes) { + UPB_ASSERT((size_t)(e->limit - e->ptr) >= bytes); + e->ptr += bytes; +} + +/* Call when all of the bytes for a handler have been written. Flushes the + * bytes if possible and necessary, returning false if this failed. */ +static bool commit(upb_pb_encoder *e) { + if (!e->top) { + /* We aren't inside a delimited region. Flush our accumulated bytes to + * the output. + * + * TODO(haberman): in the future we may want to delay flushing for + * efficiency reasons. */ + putbuf(e, e->buf, e->ptr - e->buf); + e->ptr = e->buf; + } + + return true; +} + +/* Writes the given bytes to the buffer, handling reserve/advance. */ +static bool encode_bytes(upb_pb_encoder *e, const void *data, size_t len) { + if (!reserve(e, len)) { + return false; + } + + memcpy(e->ptr, data, len); + encoder_advance(e, len); + return true; +} + +/* Finish the current run by adding the run totals to the segment and message + * length. */ +static void accumulate(upb_pb_encoder *e) { + size_t run_len; + UPB_ASSERT(e->ptr >= e->runbegin); + run_len = e->ptr - e->runbegin; + e->segptr->seglen += run_len; + top(e)->msglen += run_len; + e->runbegin = e->ptr; +} + +/* Call to indicate the start of delimited region for which the full length is + * not yet known. All data will be buffered until the length is known. + * Delimited regions may be nested; their lengths will all be tracked properly. */ +static bool start_delim(upb_pb_encoder *e) { + if (e->top) { + /* We are already buffering, advance to the next segment and push it on the + * stack. */ + accumulate(e); + + if (++e->top == e->stacklimit) { + /* TODO(haberman): grow stack? */ + return false; + } + + if (++e->segptr == e->seglimit) { + /* Grow segment buffer. */ + size_t old_size = + (e->seglimit - e->segbuf) * sizeof(upb_pb_encoder_segment); + size_t new_size = old_size * 2; + upb_pb_encoder_segment *new_buf = + upb_arena_realloc(e->arena, e->segbuf, old_size, new_size); + + if (new_buf == NULL) { + return false; + } + + e->segptr = new_buf + (e->segptr - e->segbuf); + e->seglimit = new_buf + (new_size / sizeof(upb_pb_encoder_segment)); + e->segbuf = new_buf; + } + } else { + /* We were previously at the top level, start buffering. */ + e->segptr = e->segbuf; + e->top = e->stack; + e->runbegin = e->ptr; + } + + *e->top = e->segptr - e->segbuf; + e->segptr->seglen = 0; + e->segptr->msglen = 0; + + return true; +} + +/* Call to indicate the end of a delimited region. We now know the length of + * the delimited region. If we are not nested inside any other delimited + * regions, we can now emit all of the buffered data we accumulated. */ +static bool end_delim(upb_pb_encoder *e) { + size_t msglen; + accumulate(e); + msglen = top(e)->msglen; + + if (e->top == e->stack) { + /* All lengths are now available, emit all buffered data. */ + char buf[UPB_PB_VARINT_MAX_LEN]; + upb_pb_encoder_segment *s; + const char *ptr = e->buf; + for (s = e->segbuf; s <= e->segptr; s++) { + size_t lenbytes = upb_vencode64(s->msglen, buf); + putbuf(e, buf, lenbytes); + putbuf(e, ptr, s->seglen); + ptr += s->seglen; + } + + e->ptr = e->buf; + e->top = NULL; + } else { + /* Need to keep buffering; propagate length info into enclosing + * submessages. */ + --e->top; + top(e)->msglen += msglen + upb_varint_size(msglen); + } + + return true; +} + + +/* tag_t **********************************************************************/ + +/* A precomputed (pre-encoded) tag and length. */ + +typedef struct { + uint8_t bytes; + char tag[7]; +} tag_t; + +/* Allocates a new tag for this field, and sets it in these handlerattr. */ +static void new_tag(upb_handlers *h, const upb_fielddef *f, upb_wiretype_t wt, + upb_handlerattr *attr) { + uint32_t n = upb_fielddef_number(f); + + tag_t *tag = upb_gmalloc(sizeof(tag_t)); + tag->bytes = upb_vencode64((n << 3) | wt, tag->tag); + + attr->handler_data = tag; + upb_handlers_addcleanup(h, tag, upb_gfree); +} + +static bool encode_tag(upb_pb_encoder *e, const tag_t *tag) { + return encode_bytes(e, tag->tag, tag->bytes); +} + + +/* encoding of wire types *****************************************************/ + +static bool encode_fixed64(upb_pb_encoder *e, uint64_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return encode_bytes(e, &val, sizeof(uint64_t)); +} + +static bool encode_fixed32(upb_pb_encoder *e, uint32_t val) { + /* TODO(haberman): byte-swap for big endian. */ + return encode_bytes(e, &val, sizeof(uint32_t)); +} + +static bool encode_varint(upb_pb_encoder *e, uint64_t val) { + if (!reserve(e, UPB_PB_VARINT_MAX_LEN)) { + return false; + } + + encoder_advance(e, upb_vencode64(val, e->ptr)); + return true; +} + +static uint64_t dbl2uint64(double d) { + uint64_t ret; + memcpy(&ret, &d, sizeof(uint64_t)); + return ret; +} + +static uint32_t flt2uint32(float d) { + uint32_t ret; + memcpy(&ret, &d, sizeof(uint32_t)); + return ret; +} + + +/* encoding of proto types ****************************************************/ + +static bool startmsg(void *c, const void *hd) { + upb_pb_encoder *e = c; + UPB_UNUSED(hd); + if (e->depth++ == 0) { + upb_bytessink_start(e->output_, 0, &e->subc); + } + return true; +} + +static bool endmsg(void *c, const void *hd, upb_status *status) { + upb_pb_encoder *e = c; + UPB_UNUSED(hd); + UPB_UNUSED(status); + if (--e->depth == 0) { + upb_bytessink_end(e->output_); + } + return true; +} + +static void *encode_startdelimfield(void *c, const void *hd) { + bool ok = encode_tag(c, hd) && commit(c) && start_delim(c); + return ok ? c : UPB_BREAK; +} + +static bool encode_unknown(void *c, const void *hd, const char *buf, + size_t len) { + UPB_UNUSED(hd); + return encode_bytes(c, buf, len) && commit(c); +} + +static bool encode_enddelimfield(void *c, const void *hd) { + UPB_UNUSED(hd); + return end_delim(c); +} + +static void *encode_startgroup(void *c, const void *hd) { + return (encode_tag(c, hd) && commit(c)) ? c : UPB_BREAK; +} + +static bool encode_endgroup(void *c, const void *hd) { + return encode_tag(c, hd) && commit(c); +} + +static void *encode_startstr(void *c, const void *hd, size_t size_hint) { + UPB_UNUSED(size_hint); + return encode_startdelimfield(c, hd); +} + +static size_t encode_strbuf(void *c, const void *hd, const char *buf, + size_t len, const upb_bufhandle *h) { + UPB_UNUSED(hd); + UPB_UNUSED(h); + return encode_bytes(c, buf, len) ? len : 0; +} + +#define T(type, ctype, convert, encode) \ + static bool encode_scalar_##type(void *e, const void *hd, ctype val) { \ + return encode_tag(e, hd) && encode(e, (convert)(val)) && commit(e); \ + } \ + static bool encode_packed_##type(void *e, const void *hd, ctype val) { \ + UPB_UNUSED(hd); \ + return encode(e, (convert)(val)); \ + } + +T(double, double, dbl2uint64, encode_fixed64) +T(float, float, flt2uint32, encode_fixed32) +T(int64, int64_t, uint64_t, encode_varint) +T(int32, int32_t, int64_t, encode_varint) +T(fixed64, uint64_t, uint64_t, encode_fixed64) +T(fixed32, uint32_t, uint32_t, encode_fixed32) +T(bool, bool, bool, encode_varint) +T(uint32, uint32_t, uint32_t, encode_varint) +T(uint64, uint64_t, uint64_t, encode_varint) +T(enum, int32_t, uint32_t, encode_varint) +T(sfixed32, int32_t, uint32_t, encode_fixed32) +T(sfixed64, int64_t, uint64_t, encode_fixed64) +T(sint32, int32_t, upb_zzenc_32, encode_varint) +T(sint64, int64_t, upb_zzenc_64, encode_varint) + +#undef T + + +/* code to build the handlers *************************************************/ + +#include +static void newhandlers_callback(const void *closure, upb_handlers *h) { + const upb_msgdef *m; + upb_msg_field_iter i; + + UPB_UNUSED(closure); + + upb_handlers_setstartmsg(h, startmsg, NULL); + upb_handlers_setendmsg(h, endmsg, NULL); + upb_handlers_setunknown(h, encode_unknown, NULL); + + m = upb_handlers_msgdef(h); + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + bool packed = upb_fielddef_isseq(f) && upb_fielddef_isprimitive(f) && + upb_fielddef_packed(f); + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + upb_wiretype_t wt = + packed ? UPB_WIRE_TYPE_DELIMITED + : upb_pb_native_wire_types[upb_fielddef_descriptortype(f)]; + + /* Pre-encode the tag for this field. */ + new_tag(h, f, wt, &attr); + + if (packed) { + upb_handlers_setstartseq(h, f, encode_startdelimfield, &attr); + upb_handlers_setendseq(h, f, encode_enddelimfield, &attr); + } + +#define T(upper, lower, upbtype) \ + case UPB_DESCRIPTOR_TYPE_##upper: \ + if (packed) { \ + upb_handlers_set##upbtype(h, f, encode_packed_##lower, &attr); \ + } else { \ + upb_handlers_set##upbtype(h, f, encode_scalar_##lower, &attr); \ + } \ + break; + + switch (upb_fielddef_descriptortype(f)) { + T(DOUBLE, double, double); + T(FLOAT, float, float); + T(INT64, int64, int64); + T(INT32, int32, int32); + T(FIXED64, fixed64, uint64); + T(FIXED32, fixed32, uint32); + T(BOOL, bool, bool); + T(UINT32, uint32, uint32); + T(UINT64, uint64, uint64); + T(ENUM, enum, int32); + T(SFIXED32, sfixed32, int32); + T(SFIXED64, sfixed64, int64); + T(SINT32, sint32, int32); + T(SINT64, sint64, int64); + case UPB_DESCRIPTOR_TYPE_STRING: + case UPB_DESCRIPTOR_TYPE_BYTES: + upb_handlers_setstartstr(h, f, encode_startstr, &attr); + upb_handlers_setendstr(h, f, encode_enddelimfield, &attr); + upb_handlers_setstring(h, f, encode_strbuf, &attr); + break; + case UPB_DESCRIPTOR_TYPE_MESSAGE: + upb_handlers_setstartsubmsg(h, f, encode_startdelimfield, &attr); + upb_handlers_setendsubmsg(h, f, encode_enddelimfield, &attr); + break; + case UPB_DESCRIPTOR_TYPE_GROUP: { + /* Endgroup takes a different tag (wire_type = END_GROUP). */ + upb_handlerattr attr2 = UPB_HANDLERATTR_INIT; + new_tag(h, f, UPB_WIRE_TYPE_END_GROUP, &attr2); + + upb_handlers_setstartsubmsg(h, f, encode_startgroup, &attr); + upb_handlers_setendsubmsg(h, f, encode_endgroup, &attr2); + + break; + } + } + +#undef T + } +} + +void upb_pb_encoder_reset(upb_pb_encoder *e) { + e->segptr = NULL; + e->top = NULL; + e->depth = 0; +} + + +/* public API *****************************************************************/ + +upb_handlercache *upb_pb_encoder_newcache(void) { + return upb_handlercache_new(newhandlers_callback, NULL); +} + +upb_pb_encoder *upb_pb_encoder_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output) { + const size_t initial_bufsize = 256; + const size_t initial_segbufsize = 16; + /* TODO(haberman): make this configurable. */ + const size_t stack_size = 64; +#ifndef NDEBUG + const size_t size_before = upb_arena_bytesallocated(arena); +#endif + + upb_pb_encoder *e = upb_arena_malloc(arena, sizeof(upb_pb_encoder)); + if (!e) return NULL; + + e->buf = upb_arena_malloc(arena, initial_bufsize); + e->segbuf = upb_arena_malloc(arena, initial_segbufsize * sizeof(*e->segbuf)); + e->stack = upb_arena_malloc(arena, stack_size * sizeof(*e->stack)); + + if (!e->buf || !e->segbuf || !e->stack) { + return NULL; + } + + e->limit = e->buf + initial_bufsize; + e->seglimit = e->segbuf + initial_segbufsize; + e->stacklimit = e->stack + stack_size; + + upb_pb_encoder_reset(e); + upb_sink_reset(&e->input_, h, e); + + e->arena = arena; + e->output_ = output; + e->subc = output.closure; + e->ptr = e->buf; + + /* If this fails, increase the value in encoder.h. */ + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= + UPB_PB_ENCODER_SIZE); + return e; +} + +upb_sink upb_pb_encoder_input(upb_pb_encoder *e) { return e->input_; } diff --git a/upb/pb/encoder.h b/upb/pb/encoder.h new file mode 100644 index 00000000000..f125b372188 --- /dev/null +++ b/upb/pb/encoder.h @@ -0,0 +1,83 @@ +/* +** upb::pb::Encoder (upb_pb_encoder) +** +** Implements a set of upb_handlers that write protobuf data to the binary wire +** format. +** +** This encoder implementation does not have any access to any out-of-band or +** precomputed lengths for submessages, so it must buffer submessages internally +** before it can emit the first byte. +*/ + +#ifndef UPB_ENCODER_H_ +#define UPB_ENCODER_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace pb { +class EncoderPtr; +} /* namespace pb */ +} /* namespace upb */ +#endif + +#define UPB_PBENCODER_MAX_NESTING 100 + +/* upb_pb_encoder *************************************************************/ + +/* Preallocation hint: decoder won't allocate more bytes than this when first + * constructed. This hint may be an overestimate for some build configurations. + * But if the decoder library is upgraded without recompiling the application, + * it may be an underestimate. */ +#define UPB_PB_ENCODER_SIZE 784 + +struct upb_pb_encoder; +typedef struct upb_pb_encoder upb_pb_encoder; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_sink upb_pb_encoder_input(upb_pb_encoder *p); +upb_pb_encoder* upb_pb_encoder_create(upb_arena* a, const upb_handlers* h, + upb_bytessink output); + +/* Lazily builds and caches handlers that will push encoded data to a bytessink. + * Any msgdef objects used with this object must outlive it. */ +upb_handlercache *upb_pb_encoder_newcache(void); + +#ifdef __cplusplus +} /* extern "C" { */ + +class upb::pb::EncoderPtr { + public: + EncoderPtr(upb_pb_encoder* ptr) : ptr_(ptr) {} + + upb_pb_encoder* ptr() { return ptr_; } + + /* Creates a new encoder in the given environment. The Handlers must have + * come from NewHandlers() below. */ + static EncoderPtr Create(Arena* arena, const Handlers* handlers, + BytesSink output) { + return EncoderPtr( + upb_pb_encoder_create(arena->ptr(), handlers, output.sink())); + } + + /* The input to the encoder. */ + upb::Sink input() { return upb_pb_encoder_input(ptr()); } + + /* Creates a new set of handlers for this MessageDef. */ + static HandlerCache NewCache() { + return HandlerCache(upb_pb_encoder_newcache()); + } + + static const size_t kSize = UPB_PB_ENCODER_SIZE; + + private: + upb_pb_encoder* ptr_; +}; + +#endif /* __cplusplus */ + +#endif /* UPB_ENCODER_H_ */ diff --git a/upb/pb/make-gdb-script.rb b/upb/pb/make-gdb-script.rb new file mode 100755 index 00000000000..3895597887d --- /dev/null +++ b/upb/pb/make-gdb-script.rb @@ -0,0 +1,36 @@ +#!/usr/bin/ruby + +puts "set width 0 +set height 0 +set verbose off\n\n" + +IO.popen("nm -S /tmp/upb-jit-code.so").each_line { |line| + # Input lines look like this: + # 000000000000575a T X.0x10.OP_CHECKDELIM + # + # For each one we want to emit a command that looks like: + # b X.0x10.OP_CHECKDELIM + # commands + # silent + # printf "buf_ofs=%d data_rem=%d delim_rem=%d X.0x10.OP_CHECKDELIM\n", $rbx - (long)((upb_pbdecoder*)($r15))->buf, $r12 - $rbx, $rbp - $rbx + # continue + # end + + parts = line.split + next if parts[1] != "T" + sym = parts[2] + next if sym !~ /X\./; + if sym =~ /OP_/ then + printcmd = "printf \"buf_ofs=%d data_rem=%d delim_rem=%d #{sym}\\n\", $rbx - (long)((upb_pbdecoder*)($r15))->buf, $r12 - $rbx, $rbp - $rbx" + elsif sym =~ /enterjit/ then + printcmd = "printf \"#{sym} bytes=%d\\n\", $rcx" + else + printcmd = "printf \"#{sym}\\n\"" + end + puts "b #{sym} +commands + silent + #{printcmd} + continue +end\n\n" +} diff --git a/upb/pb/textprinter.c b/upb/pb/textprinter.c new file mode 100644 index 00000000000..0760173e43c --- /dev/null +++ b/upb/pb/textprinter.c @@ -0,0 +1,340 @@ +/* + * upb::pb::TextPrinter + * + * OPT: This is not optimized at all. It uses printf() which parses the format + * string every time, and it allocates memory for every put. + */ + +#include "upb/pb/textprinter.h" + +#include +#include +#include +#include +#include +#include + +#include "upb/sink.h" + +#include "upb/port_def.inc" + +struct upb_textprinter { + upb_sink input_; + upb_bytessink output_; + int indent_depth_; + bool single_line_; + void *subc; +}; + +#define CHECK(x) if ((x) < 0) goto err; + +static const char *shortname(const char *longname) { + const char *last = strrchr(longname, '.'); + return last ? last + 1 : longname; +} + +static int indent(upb_textprinter *p) { + int i; + if (!p->single_line_) + for (i = 0; i < p->indent_depth_; i++) + upb_bytessink_putbuf(p->output_, p->subc, " ", 2, NULL); + return 0; +} + +static int endfield(upb_textprinter *p) { + const char ch = (p->single_line_ ? ' ' : '\n'); + upb_bytessink_putbuf(p->output_, p->subc, &ch, 1, NULL); + return 0; +} + +static int putescaped(upb_textprinter *p, const char *buf, size_t len, + bool preserve_utf8) { + /* Based on CEscapeInternal() from Google's protobuf release. */ + char dstbuf[4096], *dst = dstbuf, *dstend = dstbuf + sizeof(dstbuf); + const char *end = buf + len; + + /* I think hex is prettier and more useful, but proto2 uses octal; should + * investigate whether it can parse hex also. */ + const bool use_hex = false; + bool last_hex_escape = false; /* true if last output char was \xNN */ + + for (; buf < end; buf++) { + bool is_hex_escape; + + if (dstend - dst < 4) { + upb_bytessink_putbuf(p->output_, p->subc, dstbuf, dst - dstbuf, NULL); + dst = dstbuf; + } + + is_hex_escape = false; + switch (*buf) { + case '\n': *(dst++) = '\\'; *(dst++) = 'n'; break; + case '\r': *(dst++) = '\\'; *(dst++) = 'r'; break; + case '\t': *(dst++) = '\\'; *(dst++) = 't'; break; + case '\"': *(dst++) = '\\'; *(dst++) = '\"'; break; + case '\'': *(dst++) = '\\'; *(dst++) = '\''; break; + case '\\': *(dst++) = '\\'; *(dst++) = '\\'; break; + default: + /* Note that if we emit \xNN and the buf character after that is a hex + * digit then that digit must be escaped too to prevent it being + * interpreted as part of the character code by C. */ + if ((!preserve_utf8 || (uint8_t)*buf < 0x80) && + (!isprint(*buf) || (last_hex_escape && isxdigit(*buf)))) { + sprintf(dst, (use_hex ? "\\x%02x" : "\\%03o"), (uint8_t)*buf); + is_hex_escape = use_hex; + dst += 4; + } else { + *(dst++) = *buf; break; + } + } + last_hex_escape = is_hex_escape; + } + /* Flush remaining data. */ + upb_bytessink_putbuf(p->output_, p->subc, dstbuf, dst - dstbuf, NULL); + return 0; +} + +bool putf(upb_textprinter *p, const char *fmt, ...) { + va_list args; + va_list args_copy; + char *str; + int written; + int len; + bool ok; + + va_start(args, fmt); + + /* Run once to get the length of the string. */ + _upb_va_copy(args_copy, args); + len = _upb_vsnprintf(NULL, 0, fmt, args_copy); + va_end(args_copy); + + /* + 1 for NULL terminator (vsprintf() requires it even if we don't). */ + str = upb_gmalloc(len + 1); + if (!str) return false; + written = vsprintf(str, fmt, args); + va_end(args); + UPB_ASSERT(written == len); + + ok = upb_bytessink_putbuf(p->output_, p->subc, str, len, NULL); + upb_gfree(str); + return ok; +} + + +/* handlers *******************************************************************/ + +static bool textprinter_startmsg(void *c, const void *hd) { + upb_textprinter *p = c; + UPB_UNUSED(hd); + if (p->indent_depth_ == 0) { + upb_bytessink_start(p->output_, 0, &p->subc); + } + return true; +} + +static bool textprinter_endmsg(void *c, const void *hd, upb_status *s) { + upb_textprinter *p = c; + UPB_UNUSED(hd); + UPB_UNUSED(s); + if (p->indent_depth_ == 0) { + upb_bytessink_end(p->output_); + } + return true; +} + +#define TYPE(name, ctype, fmt) \ + static bool textprinter_put ## name(void *closure, const void *handler_data, \ + ctype val) { \ + upb_textprinter *p = closure; \ + const upb_fielddef *f = handler_data; \ + CHECK(indent(p)); \ + putf(p, "%s: " fmt, upb_fielddef_name(f), val); \ + CHECK(endfield(p)); \ + return true; \ + err: \ + return false; \ +} + +static bool textprinter_putbool(void *closure, const void *handler_data, + bool val) { + upb_textprinter *p = closure; + const upb_fielddef *f = handler_data; + CHECK(indent(p)); + putf(p, "%s: %s", upb_fielddef_name(f), val ? "true" : "false"); + CHECK(endfield(p)); + return true; +err: + return false; +} + +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY_MACROVAL(x) STRINGIFY_HELPER(x) + +TYPE(int32, int32_t, "%" PRId32) +TYPE(int64, int64_t, "%" PRId64) +TYPE(uint32, uint32_t, "%" PRIu32) +TYPE(uint64, uint64_t, "%" PRIu64) +TYPE(float, float, "%." STRINGIFY_MACROVAL(FLT_DIG) "g") +TYPE(double, double, "%." STRINGIFY_MACROVAL(DBL_DIG) "g") + +#undef TYPE + +/* Output a symbolic value from the enum if found, else just print as int32. */ +static bool textprinter_putenum(void *closure, const void *handler_data, + int32_t val) { + upb_textprinter *p = closure; + const upb_fielddef *f = handler_data; + const upb_enumdef *enum_def = upb_fielddef_enumsubdef(f); + const char *label = upb_enumdef_iton(enum_def, val); + if (label) { + indent(p); + putf(p, "%s: %s", upb_fielddef_name(f), label); + endfield(p); + } else { + if (!textprinter_putint32(closure, handler_data, val)) + return false; + } + return true; +} + +static void *textprinter_startstr(void *closure, const void *handler_data, + size_t size_hint) { + upb_textprinter *p = closure; + const upb_fielddef *f = handler_data; + UPB_UNUSED(size_hint); + indent(p); + putf(p, "%s: \"", upb_fielddef_name(f)); + return p; +} + +static bool textprinter_endstr(void *closure, const void *handler_data) { + upb_textprinter *p = closure; + UPB_UNUSED(handler_data); + putf(p, "\""); + endfield(p); + return true; +} + +static size_t textprinter_putstr(void *closure, const void *hd, const char *buf, + size_t len, const upb_bufhandle *handle) { + upb_textprinter *p = closure; + const upb_fielddef *f = hd; + UPB_UNUSED(handle); + CHECK(putescaped(p, buf, len, upb_fielddef_type(f) == UPB_TYPE_STRING)); + return len; +err: + return 0; +} + +static void *textprinter_startsubmsg(void *closure, const void *handler_data) { + upb_textprinter *p = closure; + const char *name = handler_data; + CHECK(indent(p)); + putf(p, "%s {%c", name, p->single_line_ ? ' ' : '\n'); + p->indent_depth_++; + return p; +err: + return UPB_BREAK; +} + +static bool textprinter_endsubmsg(void *closure, const void *handler_data) { + upb_textprinter *p = closure; + UPB_UNUSED(handler_data); + p->indent_depth_--; + CHECK(indent(p)); + upb_bytessink_putbuf(p->output_, p->subc, "}", 1, NULL); + CHECK(endfield(p)); + return true; +err: + return false; +} + +static void onmreg(const void *c, upb_handlers *h) { + const upb_msgdef *m = upb_handlers_msgdef(h); + upb_msg_field_iter i; + UPB_UNUSED(c); + + upb_handlers_setstartmsg(h, textprinter_startmsg, NULL); + upb_handlers_setendmsg(h, textprinter_endmsg, NULL); + + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + attr.handler_data = f; + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, f, textprinter_putint32, &attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, f, textprinter_putint64, &attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, f, textprinter_putuint32, &attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, f, textprinter_putuint64, &attr); + break; + case UPB_TYPE_FLOAT: + upb_handlers_setfloat(h, f, textprinter_putfloat, &attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, f, textprinter_putdouble, &attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, f, textprinter_putbool, &attr); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + upb_handlers_setstartstr(h, f, textprinter_startstr, &attr); + upb_handlers_setstring(h, f, textprinter_putstr, &attr); + upb_handlers_setendstr(h, f, textprinter_endstr, &attr); + break; + case UPB_TYPE_MESSAGE: { + const char *name = + upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_GROUP + ? shortname(upb_msgdef_fullname(upb_fielddef_msgsubdef(f))) + : upb_fielddef_name(f); + attr.handler_data = name; + upb_handlers_setstartsubmsg(h, f, textprinter_startsubmsg, &attr); + upb_handlers_setendsubmsg(h, f, textprinter_endsubmsg, &attr); + break; + } + case UPB_TYPE_ENUM: + upb_handlers_setint32(h, f, textprinter_putenum, &attr); + break; + } + } +} + +static void textprinter_reset(upb_textprinter *p, bool single_line) { + p->single_line_ = single_line; + p->indent_depth_ = 0; +} + + +/* Public API *****************************************************************/ + +upb_textprinter *upb_textprinter_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output) { + upb_textprinter *p = upb_arena_malloc(arena, sizeof(upb_textprinter)); + if (!p) return NULL; + + p->output_ = output; + upb_sink_reset(&p->input_, h, p); + textprinter_reset(p, false); + + return p; +} + +upb_handlercache *upb_textprinter_newcache(void) { + return upb_handlercache_new(&onmreg, NULL); +} + +upb_sink upb_textprinter_input(upb_textprinter *p) { return p->input_; } + +void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line) { + p->single_line_ = single_line; +} diff --git a/upb/pb/textprinter.h b/upb/pb/textprinter.h new file mode 100644 index 00000000000..7e20d7521bf --- /dev/null +++ b/upb/pb/textprinter.h @@ -0,0 +1,69 @@ +/* +** upb::pb::TextPrinter (upb_textprinter) +** +** Handlers for writing to protobuf text format. +*/ + +#ifndef UPB_TEXT_H_ +#define UPB_TEXT_H_ + +#include "upb/sink.h" + +#ifdef __cplusplus +namespace upb { +namespace pb { +class TextPrinterPtr; +} /* namespace pb */ +} /* namespace upb */ +#endif + +/* upb_textprinter ************************************************************/ + +struct upb_textprinter; +typedef struct upb_textprinter upb_textprinter; + +#ifdef __cplusplus +extern "C" { +#endif + +/* C API. */ +upb_textprinter *upb_textprinter_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output); +void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line); +upb_sink upb_textprinter_input(upb_textprinter *p); +upb_handlercache *upb_textprinter_newcache(void); + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::pb::TextPrinterPtr { + public: + TextPrinterPtr(upb_textprinter* ptr) : ptr_(ptr) {} + + /* The given handlers must have come from NewHandlers(). It must outlive the + * TextPrinter. */ + static TextPrinterPtr Create(Arena *arena, upb::HandlersPtr *handlers, + BytesSink output) { + return TextPrinterPtr( + upb_textprinter_create(arena->ptr(), handlers->ptr(), output.sink())); + } + + void SetSingleLineMode(bool single_line) { + upb_textprinter_setsingleline(ptr_, single_line); + } + + Sink input() { return upb_textprinter_input(ptr_); } + + /* If handler caching becomes a requirement we can add a code cache as in + * decoder.h */ + static HandlerCache NewCache() { + return HandlerCache(upb_textprinter_newcache()); + } + + private: + upb_textprinter* ptr_; +}; + +#endif + +#endif /* UPB_TEXT_H_ */ diff --git a/upb/pb/varint.c b/upb/pb/varint.c new file mode 100644 index 00000000000..90f58a138fc --- /dev/null +++ b/upb/pb/varint.c @@ -0,0 +1,74 @@ + +#include "upb/pb/varint.int.h" + +/* Index is descriptor type. */ +const uint8_t upb_pb_native_wire_types[] = { + UPB_WIRE_TYPE_END_GROUP, /* ENDGROUP */ + UPB_WIRE_TYPE_64BIT, /* DOUBLE */ + UPB_WIRE_TYPE_32BIT, /* FLOAT */ + UPB_WIRE_TYPE_VARINT, /* INT64 */ + UPB_WIRE_TYPE_VARINT, /* UINT64 */ + UPB_WIRE_TYPE_VARINT, /* INT32 */ + UPB_WIRE_TYPE_64BIT, /* FIXED64 */ + UPB_WIRE_TYPE_32BIT, /* FIXED32 */ + UPB_WIRE_TYPE_VARINT, /* BOOL */ + UPB_WIRE_TYPE_DELIMITED, /* STRING */ + UPB_WIRE_TYPE_START_GROUP, /* GROUP */ + UPB_WIRE_TYPE_DELIMITED, /* MESSAGE */ + UPB_WIRE_TYPE_DELIMITED, /* BYTES */ + UPB_WIRE_TYPE_VARINT, /* UINT32 */ + UPB_WIRE_TYPE_VARINT, /* ENUM */ + UPB_WIRE_TYPE_32BIT, /* SFIXED32 */ + UPB_WIRE_TYPE_64BIT, /* SFIXED64 */ + UPB_WIRE_TYPE_VARINT, /* SINT32 */ + UPB_WIRE_TYPE_VARINT, /* SINT64 */ +}; + +/* A basic branch-based decoder, uses 32-bit values to get good performance + * on 32-bit architectures (but performs well on 64-bits also). + * This scheme comes from the original Google Protobuf implementation + * (proto2). */ +upb_decoderet upb_vdecode_max8_branch32(upb_decoderet r) { + upb_decoderet err = {NULL, 0}; + const char *p = r.p; + uint32_t low = (uint32_t)r.val; + uint32_t high = 0; + uint32_t b; + b = *(p++); low |= (b & 0x7fU) << 14; if (!(b & 0x80)) goto done; + b = *(p++); low |= (b & 0x7fU) << 21; if (!(b & 0x80)) goto done; + b = *(p++); low |= (b & 0x7fU) << 28; + high = (b & 0x7fU) >> 4; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 3; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 10; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 17; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 24; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 31; if (!(b & 0x80)) goto done; + return err; + +done: + r.val = ((uint64_t)high << 32) | low; + r.p = p; + return r; +} + +/* Like the previous, but uses 64-bit values. */ +upb_decoderet upb_vdecode_max8_branch64(upb_decoderet r) { + const char *p = r.p; + uint64_t val = r.val; + uint64_t b; + upb_decoderet err = {NULL, 0}; + b = *(p++); val |= (b & 0x7fU) << 14; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 21; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 28; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 35; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 42; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 49; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 56; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 63; if (!(b & 0x80)) goto done; + return err; + +done: + r.val = val; + r.p = p; + return r; +} diff --git a/upb/pb/varint.int.h b/upb/pb/varint.int.h new file mode 100644 index 00000000000..ff1ca661491 --- /dev/null +++ b/upb/pb/varint.int.h @@ -0,0 +1,164 @@ +/* +** A number of routines for varint manipulation (we keep them all around to +** have multiple approaches available for benchmarking). +*/ + +#ifndef UPB_VARINT_DECODER_H_ +#define UPB_VARINT_DECODER_H_ + +#include +#include +#include +#include "upb/upb.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +#define UPB_MAX_WIRE_TYPE 5 + +/* The maximum number of bytes that it takes to encode a 64-bit varint. */ +#define UPB_PB_VARINT_MAX_LEN 10 + +/* Array of the "native" (ie. non-packed-repeated) wire type for the given a + * descriptor type (upb_descriptortype_t). */ +extern const uint8_t upb_pb_native_wire_types[]; + +UPB_INLINE uint64_t byteswap64(uint64_t val) +{ + return ((((val) & 0xff00000000000000ull) >> 56) + | (((val) & 0x00ff000000000000ull) >> 40) + | (((val) & 0x0000ff0000000000ull) >> 24) + | (((val) & 0x000000ff00000000ull) >> 8) + | (((val) & 0x00000000ff000000ull) << 8) + | (((val) & 0x0000000000ff0000ull) << 24) + | (((val) & 0x000000000000ff00ull) << 40) + | (((val) & 0x00000000000000ffull) << 56)); +} + +/* Zig-zag encoding/decoding **************************************************/ + +UPB_INLINE int32_t upb_zzdec_32(uint32_t n) { + return (n >> 1) ^ -(int32_t)(n & 1); +} +UPB_INLINE int64_t upb_zzdec_64(uint64_t n) { + return (n >> 1) ^ -(int64_t)(n & 1); +} +UPB_INLINE uint32_t upb_zzenc_32(int32_t n) { + return ((uint32_t)n << 1) ^ (n >> 31); +} +UPB_INLINE uint64_t upb_zzenc_64(int64_t n) { + return ((uint64_t)n << 1) ^ (n >> 63); +} + +/* Decoding *******************************************************************/ + +/* All decoding functions return this struct by value. */ +typedef struct { + const char *p; /* NULL if the varint was unterminated. */ + uint64_t val; +} upb_decoderet; + +UPB_INLINE upb_decoderet upb_decoderet_make(const char *p, uint64_t val) { + upb_decoderet ret; + ret.p = p; + ret.val = val; + return ret; +} + +upb_decoderet upb_vdecode_max8_branch32(upb_decoderet r); +upb_decoderet upb_vdecode_max8_branch64(upb_decoderet r); + +/* Template for a function that checks the first two bytes with branching + * and dispatches 2-10 bytes with a separate function. Note that this may read + * up to 10 bytes, so it must not be used unless there are at least ten bytes + * left in the buffer! */ +#define UPB_VARINT_DECODER_CHECK2(name, decode_max8_function) \ +UPB_INLINE upb_decoderet upb_vdecode_check2_ ## name(const char *_p) { \ + uint8_t *p = (uint8_t*)_p; \ + upb_decoderet r; \ + if ((*p & 0x80) == 0) { \ + /* Common case: one-byte varint. */ \ + return upb_decoderet_make(_p + 1, *p & 0x7fU); \ + } \ + r = upb_decoderet_make(_p + 2, (*p & 0x7fU) | ((*(p + 1) & 0x7fU) << 7)); \ + if ((*(p + 1) & 0x80) == 0) { \ + /* Two-byte varint. */ \ + return r; \ + } \ + /* Longer varint, fallback to out-of-line function. */ \ + return decode_max8_function(r); \ +} + +UPB_VARINT_DECODER_CHECK2(branch32, upb_vdecode_max8_branch32) +UPB_VARINT_DECODER_CHECK2(branch64, upb_vdecode_max8_branch64) +#undef UPB_VARINT_DECODER_CHECK2 + +/* Our canonical functions for decoding varints, based on the currently + * favored best-performing implementations. */ +UPB_INLINE upb_decoderet upb_vdecode_fast(const char *p) { + if (sizeof(long) == 8) + return upb_vdecode_check2_branch64(p); + else + return upb_vdecode_check2_branch32(p); +} + + +/* Encoding *******************************************************************/ + +UPB_INLINE int upb_value_size(uint64_t val) { +#ifdef __GNUC__ + int high_bit = 63 - __builtin_clzll(val); /* 0-based, undef if val == 0. */ +#else + int high_bit = 0; + uint64_t tmp = val; + while(tmp >>= 1) high_bit++; +#endif + return val == 0 ? 1 : high_bit / 8 + 1; +} + +/* Encodes a 64-bit varint into buf (which must be >=UPB_PB_VARINT_MAX_LEN + * bytes long), returning how many bytes were used. + * + * TODO: benchmark and optimize if necessary. */ +UPB_INLINE size_t upb_vencode64(uint64_t val, char *buf) { + size_t i; + if (val == 0) { buf[0] = 0; return 1; } + i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +UPB_INLINE size_t upb_varint_size(uint64_t val) { + char buf[UPB_PB_VARINT_MAX_LEN]; + return upb_vencode64(val, buf); +} + +/* Encodes a 32-bit varint, *not* sign-extended. */ +UPB_INLINE uint64_t upb_vencode32(uint32_t val) { + char buf[UPB_PB_VARINT_MAX_LEN]; + size_t bytes = upb_vencode64(val, buf); + uint64_t ret = 0; + UPB_ASSERT(bytes <= 5); + memcpy(&ret, buf, bytes); +#ifdef UPB_BIG_ENDIAN + ret = byteswap64(ret); +#endif + UPB_ASSERT(ret <= 0xffffffffffU); + return ret; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* UPB_VARINT_DECODER_H_ */ diff --git a/upb/port.c b/upb/port.c new file mode 100644 index 00000000000..023f7dc0ce1 --- /dev/null +++ b/upb/port.c @@ -0,0 +1,27 @@ + +#include "upb/upb.h" +#include "upb/port_def.inc" + +#ifdef UPB_MSVC_VSNPRINTF +/* Visual C++ earlier than 2015 doesn't have standard C99 snprintf and + * vsnprintf. To support them, missing functions are manually implemented + * using the existing secure functions. */ +int msvc_vsnprintf(char* s, size_t n, const char* format, va_list arg) { + if (!s) { + return _vscprintf(format, arg); + } + int ret = _vsnprintf_s(s, n, _TRUNCATE, format, arg); + if (ret < 0) { + ret = _vscprintf(format, arg); + } + return ret; +} + +int msvc_snprintf(char* s, size_t n, const char* format, ...) { + va_list arg; + va_start(arg, format); + int ret = msvc_vsnprintf(s, n, format, arg); + va_end(arg); + return ret; +} +#endif diff --git a/upb/port_def.inc b/upb/port_def.inc new file mode 100644 index 00000000000..a8967b36794 --- /dev/null +++ b/upb/port_def.inc @@ -0,0 +1,152 @@ +/* +* This is where we define macros used across upb. +* +* All of these macros are undef'd in port_undef.inc to avoid leaking them to +* users. +* +* The correct usage is: +* +* #include "upb/foobar.h" +* #include "upb/baz.h" +* +* // MUST be last included header. +* #include "upb/port_def.inc" +* +* // Code for this file. +* // <...> +* +* // Can be omitted for .c files, required for .h. +* #include "upb/port_undef.inc" +* +* This file is private and must not be included by users! +*/ +#ifndef UINTPTR_MAX +#error must include stdint.h first +#endif + +#if UINTPTR_MAX == 0xffffffff +#define UPB_SIZE(size32, size64) size32 +#else +#define UPB_SIZE(size32, size64) size64 +#endif + +#define UPB_FIELD_AT(msg, fieldtype, offset) \ + *(fieldtype*)((const char*)(msg) + offset) + +#define UPB_READ_ONEOF(msg, fieldtype, offset, case_offset, case_val, default) \ + UPB_FIELD_AT(msg, int, case_offset) == case_val \ + ? UPB_FIELD_AT(msg, fieldtype, offset) \ + : default + +#define UPB_WRITE_ONEOF(msg, fieldtype, offset, value, case_offset, case_val) \ + UPB_FIELD_AT(msg, int, case_offset) = case_val; \ + UPB_FIELD_AT(msg, fieldtype, offset) = value; + +/* UPB_INLINE: inline if possible, emit standalone code if required. */ +#ifdef __cplusplus +#define UPB_INLINE inline +#elif defined (__GNUC__) || defined(__clang__) +#define UPB_INLINE static __inline__ +#else +#define UPB_INLINE static +#endif + +/* Hints to the compiler about likely/unlikely branches. */ +#if defined (__GNUC__) || defined(__clang__) +#define UPB_LIKELY(x) __builtin_expect((x),1) +#define UPB_UNLIKELY(x) __builtin_expect((x),0) +#else +#define UPB_LIKELY(x) (x) +#define UPB_UNLIKELY(x) (x) +#endif + +/* Define UPB_BIG_ENDIAN manually if you're on big endian and your compiler + * doesn't provide these preprocessor symbols. */ +#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define UPB_BIG_ENDIAN +#endif + +/* Macros for function attributes on compilers that support them. */ +#ifdef __GNUC__ +#define UPB_FORCEINLINE __inline__ __attribute__((always_inline)) +#define UPB_NOINLINE __attribute__((noinline)) +#define UPB_NORETURN __attribute__((__noreturn__)) +#else /* !defined(__GNUC__) */ +#define UPB_FORCEINLINE +#define UPB_NOINLINE +#define UPB_NORETURN +#endif + +#if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L +/* C99/C++11 versions. */ +#include +#define _upb_snprintf snprintf +#define _upb_vsnprintf vsnprintf +#define _upb_va_copy(a, b) va_copy(a, b) +#elif defined(_MSC_VER) +/* Microsoft C/C++ versions. */ +#include +#include +#if _MSC_VER < 1900 +int msvc_snprintf(char* s, size_t n, const char* format, ...); +int msvc_vsnprintf(char* s, size_t n, const char* format, va_list arg); +#define UPB_MSVC_VSNPRINTF +#define _upb_snprintf msvc_snprintf +#define _upb_vsnprintf msvc_vsnprintf +#else +#define _upb_snprintf snprintf +#define _upb_vsnprintf vsnprintf +#endif +#define _upb_va_copy(a, b) va_copy(a, b) +#elif defined __GNUC__ +/* A few hacky workarounds for functions not in C89. + * For internal use only! + * TODO(haberman): fix these by including our own implementations, or finding + * another workaround. + */ +#define _upb_snprintf __builtin_snprintf +#define _upb_vsnprintf __builtin_vsnprintf +#define _upb_va_copy(a, b) __va_copy(a, b) +#else +#error Need implementations of [v]snprintf and va_copy +#endif + +#ifdef __cplusplus +#if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__) || \ + (defined(_MSC_VER) && _MSC_VER >= 1900) +/* C++11 is present */ +#else +#error upb requires C++11 for C++ support +#endif +#endif + +#define UPB_MAX(x, y) ((x) > (y) ? (x) : (y)) +#define UPB_MIN(x, y) ((x) < (y) ? (x) : (y)) + +#define UPB_UNUSED(var) (void)var + +/* UPB_ASSERT(): in release mode, we use the expression without letting it be + * evaluated. This prevents "unused variable" warnings. */ +#ifdef NDEBUG +#define UPB_ASSERT(expr) do {} while (false && (expr)) +#else +#define UPB_ASSERT(expr) assert(expr) +#endif + +/* UPB_ASSERT_DEBUGVAR(): assert that uses functions or variables that only + * exist in debug mode. This turns into regular assert. */ +#define UPB_ASSERT_DEBUGVAR(expr) assert(expr) + +#if defined(__GNUC__) || defined(__clang__) +#define UPB_UNREACHABLE() do { assert(0); __builtin_unreachable(); } while(0) +#else +#define UPB_UNREACHABLE() do { assert(0); } while(0) +#endif + +/* UPB_INFINITY representing floating-point positive infinity. */ +#include +#ifdef INFINITY +#define UPB_INFINITY INFINITY +#else +#define UPB_INFINITY (1.0 / 0.0) +#endif diff --git a/upb/port_undef.inc b/upb/port_undef.inc new file mode 100644 index 00000000000..103180b7fbd --- /dev/null +++ b/upb/port_undef.inc @@ -0,0 +1,21 @@ +/* See port_def.inc. This should #undef all macros #defined there. */ + +#undef UPB_SIZE +#undef UPB_FIELD_AT +#undef UPB_READ_ONEOF +#undef UPB_WRITE_ONEOF +#undef UPB_INLINE +#undef UPB_FORCEINLINE +#undef UPB_NOINLINE +#undef UPB_NORETURN +#undef UPB_MAX +#undef UPB_MIN +#undef UPB_UNUSED +#undef UPB_ASSERT +#undef UPB_ASSERT_DEBUGVAR +#undef UPB_UNREACHABLE +#undef UPB_INFINITY +#undef UPB_MSVC_VSNPRINTF +#undef _upb_snprintf +#undef _upb_vsnprintf +#undef _upb_va_copy diff --git a/upb/sink.c b/upb/sink.c new file mode 100644 index 00000000000..d55d258b235 --- /dev/null +++ b/upb/sink.c @@ -0,0 +1,17 @@ + +#include "upb/sink.h" + +bool upb_bufsrc_putbuf(const char *buf, size_t len, upb_bytessink sink) { + void *subc; + bool ret; + upb_bufhandle handle = UPB_BUFHANDLE_INIT; + handle.buf = buf; + ret = upb_bytessink_start(sink, len, &subc); + if (ret && len != 0) { + ret = (upb_bytessink_putbuf(sink, subc, buf, len, &handle) >= len); + } + if (ret) { + ret = upb_bytessink_end(sink); + } + return ret; +} diff --git a/upb/sink.h b/upb/sink.h new file mode 100644 index 00000000000..47d218a9c1b --- /dev/null +++ b/upb/sink.h @@ -0,0 +1,516 @@ +/* +** upb::Sink (upb_sink) +** upb::BytesSink (upb_bytessink) +** +** A upb_sink is an object that binds a upb_handlers object to some runtime +** state. It is the object that can actually receive data via the upb_handlers +** interface. +** +** Unlike upb_def and upb_handlers, upb_sink is never frozen, immutable, or +** thread-safe. You can create as many of them as you want, but each one may +** only be used in a single thread at a time. +** +** If we compare with class-based OOP, a you can think of a upb_def as an +** abstract base class, a upb_handlers as a concrete derived class, and a +** upb_sink as an object (class instance). +*/ + +#ifndef UPB_SINK_H +#define UPB_SINK_H + +#include "upb/handlers.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +namespace upb { +class BytesSink; +class Sink; +} +#endif + +/* upb_sink *******************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + const upb_handlers *handlers; + void *closure; +} upb_sink; + +#define PUTVAL(type, ctype) \ + UPB_INLINE bool upb_sink_put##type(upb_sink s, upb_selector_t sel, \ + ctype val) { \ + typedef upb_##type##_handlerfunc functype; \ + functype *func; \ + const void *hd; \ + if (!s.handlers) return true; \ + func = (functype *)upb_handlers_gethandler(s.handlers, sel, &hd); \ + if (!func) return true; \ + return func(s.closure, hd, val); \ + } + +PUTVAL(int32, int32_t) +PUTVAL(int64, int64_t) +PUTVAL(uint32, uint32_t) +PUTVAL(uint64, uint64_t) +PUTVAL(float, float) +PUTVAL(double, double) +PUTVAL(bool, bool) +#undef PUTVAL + +UPB_INLINE void upb_sink_reset(upb_sink *s, const upb_handlers *h, void *c) { + s->handlers = h; + s->closure = c; +} + +UPB_INLINE size_t upb_sink_putstring(upb_sink s, upb_selector_t sel, + const char *buf, size_t n, + const upb_bufhandle *handle) { + typedef upb_string_handlerfunc func; + func *handler; + const void *hd; + if (!s.handlers) return n; + handler = (func *)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!handler) return n; + return handler(s.closure, hd, buf, n, handle); +} + +UPB_INLINE bool upb_sink_putunknown(upb_sink s, const char *buf, size_t n) { + typedef upb_unknown_handlerfunc func; + func *handler; + const void *hd; + if (!s.handlers) return true; + handler = + (func *)upb_handlers_gethandler(s.handlers, UPB_UNKNOWN_SELECTOR, &hd); + + if (!handler) return n; + return handler(s.closure, hd, buf, n); +} + +UPB_INLINE bool upb_sink_startmsg(upb_sink s) { + typedef upb_startmsg_handlerfunc func; + func *startmsg; + const void *hd; + if (!s.handlers) return true; + startmsg = + (func *)upb_handlers_gethandler(s.handlers, UPB_STARTMSG_SELECTOR, &hd); + + if (!startmsg) return true; + return startmsg(s.closure, hd); +} + +UPB_INLINE bool upb_sink_endmsg(upb_sink s, upb_status *status) { + typedef upb_endmsg_handlerfunc func; + func *endmsg; + const void *hd; + if (!s.handlers) return true; + endmsg = + (func *)upb_handlers_gethandler(s.handlers, UPB_ENDMSG_SELECTOR, &hd); + + if (!endmsg) return true; + return endmsg(s.closure, hd, status); +} + +UPB_INLINE bool upb_sink_startseq(upb_sink s, upb_selector_t sel, + upb_sink *sub) { + typedef upb_startfield_handlerfunc func; + func *startseq; + const void *hd; + sub->closure = s.closure; + sub->handlers = s.handlers; + if (!s.handlers) return true; + startseq = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!startseq) return true; + sub->closure = startseq(s.closure, hd); + return sub->closure ? true : false; +} + +UPB_INLINE bool upb_sink_endseq(upb_sink s, upb_selector_t sel) { + typedef upb_endfield_handlerfunc func; + func *endseq; + const void *hd; + if (!s.handlers) return true; + endseq = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!endseq) return true; + return endseq(s.closure, hd); +} + +UPB_INLINE bool upb_sink_startstr(upb_sink s, upb_selector_t sel, + size_t size_hint, upb_sink *sub) { + typedef upb_startstr_handlerfunc func; + func *startstr; + const void *hd; + sub->closure = s.closure; + sub->handlers = s.handlers; + if (!s.handlers) return true; + startstr = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!startstr) return true; + sub->closure = startstr(s.closure, hd, size_hint); + return sub->closure ? true : false; +} + +UPB_INLINE bool upb_sink_endstr(upb_sink s, upb_selector_t sel) { + typedef upb_endfield_handlerfunc func; + func *endstr; + const void *hd; + if (!s.handlers) return true; + endstr = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!endstr) return true; + return endstr(s.closure, hd); +} + +UPB_INLINE bool upb_sink_startsubmsg(upb_sink s, upb_selector_t sel, + upb_sink *sub) { + typedef upb_startfield_handlerfunc func; + func *startsubmsg; + const void *hd; + sub->closure = s.closure; + if (!s.handlers) { + sub->handlers = NULL; + return true; + } + sub->handlers = upb_handlers_getsubhandlers_sel(s.handlers, sel); + startsubmsg = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!startsubmsg) return true; + sub->closure = startsubmsg(s.closure, hd); + return sub->closure ? true : false; +} + +UPB_INLINE bool upb_sink_endsubmsg(upb_sink s, upb_selector_t sel) { + typedef upb_endfield_handlerfunc func; + func *endsubmsg; + const void *hd; + if (!s.handlers) return true; + endsubmsg = (func*)upb_handlers_gethandler(s.handlers, sel, &hd); + + if (!endsubmsg) return s.closure; + return endsubmsg(s.closure, hd); +} + +#ifdef __cplusplus +} /* extern "C" */ + +/* A upb::Sink is an object that binds a upb::Handlers object to some runtime + * state. It represents an endpoint to which data can be sent. + * + * TODO(haberman): right now all of these functions take selectors. Should they + * take selectorbase instead? + * + * ie. instead of calling: + * sink->StartString(FOO_FIELD_START_STRING, ...) + * a selector base would let you say: + * sink->StartString(FOO_FIELD, ...) + * + * This would make call sites a little nicer and require emitting fewer selector + * definitions in .h files. + * + * But the current scheme has the benefit that you can retrieve a function + * pointer for any handler with handlers->GetHandler(selector), without having + * to have a separate GetHandler() function for each handler type. The JIT + * compiler uses this. To accommodate we'd have to expose a separate + * GetHandler() for every handler type. + * + * Also to ponder: selectors right now are independent of a specific Handlers + * instance. In other words, they allocate a number to every possible handler + * that *could* be registered, without knowing anything about what handlers + * *are* registered. That means that using selectors as table offsets prohibits + * us from compacting the handler table at Freeze() time. If the table is very + * sparse, this could be wasteful. + * + * Having another selector-like thing that is specific to a Handlers instance + * would allow this compacting, but then it would be impossible to write code + * ahead-of-time that can be bound to any Handlers instance at runtime. For + * example, a .proto file parser written as straight C will not know what + * Handlers it will be bound to, so when it calls sink->StartString() what + * selector will it pass? It needs a selector like we have today, that is + * independent of any particular upb::Handlers. + * + * Is there a way then to allow Handlers table compaction? */ +class upb::Sink { + public: + /* Constructor with no initialization; must be Reset() before use. */ + Sink() {} + + Sink(const Sink&) = default; + Sink& operator=(const Sink&) = default; + + Sink(const upb_sink& sink) : sink_(sink) {} + Sink &operator=(const upb_sink &sink) { + sink_ = sink; + return *this; + } + + upb_sink sink() { return sink_; } + + /* Constructs a new sink for the given frozen handlers and closure. + * + * TODO: once the Handlers know the expected closure type, verify that T + * matches it. */ + template Sink(const upb_handlers* handlers, T* closure) { + Reset(handlers, closure); + } + + upb_sink* ptr() { return &sink_; } + + /* Resets the value of the sink. */ + template void Reset(const upb_handlers* handlers, T* closure) { + upb_sink_reset(&sink_, handlers, closure); + } + + /* Returns the top-level object that is bound to this sink. + * + * TODO: once the Handlers know the expected closure type, verify that T + * matches it. */ + template T* GetObject() const { + return static_cast(sink_.closure); + } + + /* Functions for pushing data into the sink. + * + * These return false if processing should stop (either due to error or just + * to suspend). + * + * These may not be called from within one of the same sink's handlers (in + * other words, handlers are not re-entrant). */ + + /* Should be called at the start and end of every message; both the top-level + * message and submessages. This means that submessages should use the + * following sequence: + * sink->StartSubMessage(startsubmsg_selector); + * sink->StartMessage(); + * // ... + * sink->EndMessage(&status); + * sink->EndSubMessage(endsubmsg_selector); */ + bool StartMessage() { return upb_sink_startmsg(sink_); } + bool EndMessage(upb_status *status) { + return upb_sink_endmsg(sink_, status); + } + + /* Putting of individual values. These work for both repeated and + * non-repeated fields, but for repeated fields you must wrap them in + * calls to StartSequence()/EndSequence(). */ + bool PutInt32(HandlersPtr::Selector s, int32_t val) { + return upb_sink_putint32(sink_, s, val); + } + + bool PutInt64(HandlersPtr::Selector s, int64_t val) { + return upb_sink_putint64(sink_, s, val); + } + + bool PutUInt32(HandlersPtr::Selector s, uint32_t val) { + return upb_sink_putuint32(sink_, s, val); + } + + bool PutUInt64(HandlersPtr::Selector s, uint64_t val) { + return upb_sink_putuint64(sink_, s, val); + } + + bool PutFloat(HandlersPtr::Selector s, float val) { + return upb_sink_putfloat(sink_, s, val); + } + + bool PutDouble(HandlersPtr::Selector s, double val) { + return upb_sink_putdouble(sink_, s, val); + } + + bool PutBool(HandlersPtr::Selector s, bool val) { + return upb_sink_putbool(sink_, s, val); + } + + /* Putting of string/bytes values. Each string can consist of zero or more + * non-contiguous buffers of data. + * + * For StartString(), the function will write a sink for the string to "sub." + * The sub-sink must be used for any/all PutStringBuffer() calls. */ + bool StartString(HandlersPtr::Selector s, size_t size_hint, Sink* sub) { + upb_sink sub_c; + bool ret = upb_sink_startstr(sink_, s, size_hint, &sub_c); + *sub = sub_c; + return ret; + } + + size_t PutStringBuffer(HandlersPtr::Selector s, const char *buf, size_t len, + const upb_bufhandle *handle) { + return upb_sink_putstring(sink_, s, buf, len, handle); + } + + bool EndString(HandlersPtr::Selector s) { + return upb_sink_endstr(sink_, s); + } + + /* For submessage fields. + * + * For StartSubMessage(), the function will write a sink for the string to + * "sub." The sub-sink must be used for any/all handlers called within the + * submessage. */ + bool StartSubMessage(HandlersPtr::Selector s, Sink* sub) { + upb_sink sub_c; + bool ret = upb_sink_startsubmsg(sink_, s, &sub_c); + *sub = sub_c; + return ret; + } + + bool EndSubMessage(HandlersPtr::Selector s) { + return upb_sink_endsubmsg(sink_, s); + } + + /* For repeated fields of any type, the sequence of values must be wrapped in + * these calls. + * + * For StartSequence(), the function will write a sink for the string to + * "sub." The sub-sink must be used for any/all handlers called within the + * sequence. */ + bool StartSequence(HandlersPtr::Selector s, Sink* sub) { + upb_sink sub_c; + bool ret = upb_sink_startseq(sink_, s, &sub_c); + *sub = sub_c; + return ret; + } + + bool EndSequence(HandlersPtr::Selector s) { + return upb_sink_endseq(sink_, s); + } + + /* Copy and assign specifically allowed. + * We don't even bother making these members private because so many + * functions need them and this is mainly just a dumb data container anyway. + */ + + private: + upb_sink sink_; +}; + +#endif /* __cplusplus */ + +/* upb_bytessink **************************************************************/ + +typedef struct { + const upb_byteshandler *handler; + void *closure; +} upb_bytessink ; + +UPB_INLINE void upb_bytessink_reset(upb_bytessink* s, const upb_byteshandler *h, + void *closure) { + s->handler = h; + s->closure = closure; +} + +UPB_INLINE bool upb_bytessink_start(upb_bytessink s, size_t size_hint, + void **subc) { + typedef upb_startstr_handlerfunc func; + func *start; + *subc = s.closure; + if (!s.handler) return true; + start = (func *)s.handler->table[UPB_STARTSTR_SELECTOR].func; + + if (!start) return true; + *subc = start(s.closure, + s.handler->table[UPB_STARTSTR_SELECTOR].attr.handler_data, + size_hint); + return *subc != NULL; +} + +UPB_INLINE size_t upb_bytessink_putbuf(upb_bytessink s, void *subc, + const char *buf, size_t size, + const upb_bufhandle* handle) { + typedef upb_string_handlerfunc func; + func *putbuf; + if (!s.handler) return true; + putbuf = (func *)s.handler->table[UPB_STRING_SELECTOR].func; + + if (!putbuf) return true; + return putbuf(subc, s.handler->table[UPB_STRING_SELECTOR].attr.handler_data, + buf, size, handle); +} + +UPB_INLINE bool upb_bytessink_end(upb_bytessink s) { + typedef upb_endfield_handlerfunc func; + func *end; + if (!s.handler) return true; + end = (func *)s.handler->table[UPB_ENDSTR_SELECTOR].func; + + if (!end) return true; + return end(s.closure, + s.handler->table[UPB_ENDSTR_SELECTOR].attr.handler_data); +} + +#ifdef __cplusplus + +class upb::BytesSink { + public: + BytesSink() {} + + BytesSink(const BytesSink&) = default; + BytesSink& operator=(const BytesSink&) = default; + + BytesSink(const upb_bytessink& sink) : sink_(sink) {} + BytesSink &operator=(const upb_bytessink &sink) { + sink_ = sink; + return *this; + } + + upb_bytessink sink() { return sink_; } + + /* Constructs a new sink for the given frozen handlers and closure. + * + * TODO(haberman): once the Handlers know the expected closure type, verify + * that T matches it. */ + template BytesSink(const upb_byteshandler* handler, T* closure) { + upb_bytessink_reset(sink_, handler, closure); + } + + /* Resets the value of the sink. */ + template void Reset(const upb_byteshandler* handler, T* closure) { + upb_bytessink_reset(&sink_, handler, closure); + } + + bool Start(size_t size_hint, void **subc) { + return upb_bytessink_start(sink_, size_hint, subc); + } + + size_t PutBuffer(void *subc, const char *buf, size_t len, + const upb_bufhandle *handle) { + return upb_bytessink_putbuf(sink_, subc, buf, len, handle); + } + + bool End() { + return upb_bytessink_end(sink_); + } + + private: + upb_bytessink sink_; +}; + +#endif /* __cplusplus */ + +/* upb_bufsrc *****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +bool upb_bufsrc_putbuf(const char *buf, size_t len, upb_bytessink sink); + +#ifdef __cplusplus +} /* extern "C" */ + +namespace upb { +template bool PutBuffer(const T& str, BytesSink sink) { + return upb_bufsrc_putbuf(str.data(), str.size(), sink.sink()); +} +} + +#endif /* __cplusplus */ + +#include "upb/port_undef.inc" + +#endif diff --git a/upb/table.c b/upb/table.c new file mode 100644 index 00000000000..8896d217db6 --- /dev/null +++ b/upb/table.c @@ -0,0 +1,912 @@ +/* +** upb_table Implementation +** +** Implementation is heavily inspired by Lua's ltable.c. +*/ + +#include "upb/table.int.h" + +#include + +#include "upb/port_def.inc" + +#define UPB_MAXARRSIZE 16 /* 64k. */ + +/* From Chromium. */ +#define ARRAY_SIZE(x) \ + ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x]))))) + +static void upb_check_alloc(upb_table *t, upb_alloc *a) { + UPB_UNUSED(t); + UPB_UNUSED(a); + UPB_ASSERT_DEBUGVAR(t->alloc == a); +} + +static const double MAX_LOAD = 0.85; + +/* The minimum utilization of the array part of a mixed hash/array table. This + * is a speed/memory-usage tradeoff (though it's not straightforward because of + * cache effects). The lower this is, the more memory we'll use. */ +static const double MIN_DENSITY = 0.1; + +bool is_pow2(uint64_t v) { return v == 0 || (v & (v - 1)) == 0; } + +int log2ceil(uint64_t v) { + int ret = 0; + bool pow2 = is_pow2(v); + while (v >>= 1) ret++; + ret = pow2 ? ret : ret + 1; /* Ceiling. */ + return UPB_MIN(UPB_MAXARRSIZE, ret); +} + +char *upb_strdup(const char *s, upb_alloc *a) { + return upb_strdup2(s, strlen(s), a); +} + +char *upb_strdup2(const char *s, size_t len, upb_alloc *a) { + size_t n; + char *p; + + /* Prevent overflow errors. */ + if (len == SIZE_MAX) return NULL; + /* Always null-terminate, even if binary data; but don't rely on the input to + * have a null-terminating byte since it may be a raw binary buffer. */ + n = len + 1; + p = upb_malloc(a, n); + if (p) { + memcpy(p, s, len); + p[len] = 0; + } + return p; +} + +/* A type to represent the lookup key of either a strtable or an inttable. */ +typedef union { + uintptr_t num; + struct { + const char *str; + size_t len; + } str; +} lookupkey_t; + +static lookupkey_t strkey2(const char *str, size_t len) { + lookupkey_t k; + k.str.str = str; + k.str.len = len; + return k; +} + +static lookupkey_t intkey(uintptr_t key) { + lookupkey_t k; + k.num = key; + return k; +} + +typedef uint32_t hashfunc_t(upb_tabkey key); +typedef bool eqlfunc_t(upb_tabkey k1, lookupkey_t k2); + +/* Base table (shared code) ***************************************************/ + +/* For when we need to cast away const. */ +static upb_tabent *mutable_entries(upb_table *t) { + return (upb_tabent*)t->entries; +} + +static bool isfull(upb_table *t) { + if (upb_table_size(t) == 0) { + return true; + } else { + return ((double)(t->count + 1) / upb_table_size(t)) > MAX_LOAD; + } +} + +static bool init(upb_table *t, upb_ctype_t ctype, uint8_t size_lg2, + upb_alloc *a) { + size_t bytes; + + t->count = 0; + t->ctype = ctype; + t->size_lg2 = size_lg2; + t->mask = upb_table_size(t) ? upb_table_size(t) - 1 : 0; +#ifndef NDEBUG + t->alloc = a; +#endif + bytes = upb_table_size(t) * sizeof(upb_tabent); + if (bytes > 0) { + t->entries = upb_malloc(a, bytes); + if (!t->entries) return false; + memset(mutable_entries(t), 0, bytes); + } else { + t->entries = NULL; + } + return true; +} + +static void uninit(upb_table *t, upb_alloc *a) { + upb_check_alloc(t, a); + upb_free(a, mutable_entries(t)); +} + +static upb_tabent *emptyent(upb_table *t) { + upb_tabent *e = mutable_entries(t) + upb_table_size(t); + while (1) { if (upb_tabent_isempty(--e)) return e; UPB_ASSERT(e > t->entries); } +} + +static upb_tabent *getentry_mutable(upb_table *t, uint32_t hash) { + return (upb_tabent*)upb_getentry(t, hash); +} + +static const upb_tabent *findentry(const upb_table *t, lookupkey_t key, + uint32_t hash, eqlfunc_t *eql) { + const upb_tabent *e; + + if (t->size_lg2 == 0) return NULL; + e = upb_getentry(t, hash); + if (upb_tabent_isempty(e)) return NULL; + while (1) { + if (eql(e->key, key)) return e; + if ((e = e->next) == NULL) return NULL; + } +} + +static upb_tabent *findentry_mutable(upb_table *t, lookupkey_t key, + uint32_t hash, eqlfunc_t *eql) { + return (upb_tabent*)findentry(t, key, hash, eql); +} + +static bool lookup(const upb_table *t, lookupkey_t key, upb_value *v, + uint32_t hash, eqlfunc_t *eql) { + const upb_tabent *e = findentry(t, key, hash, eql); + if (e) { + if (v) { + _upb_value_setval(v, e->val.val, t->ctype); + } + return true; + } else { + return false; + } +} + +/* The given key must not already exist in the table. */ +static void insert(upb_table *t, lookupkey_t key, upb_tabkey tabkey, + upb_value val, uint32_t hash, + hashfunc_t *hashfunc, eqlfunc_t *eql) { + upb_tabent *mainpos_e; + upb_tabent *our_e; + + UPB_ASSERT(findentry(t, key, hash, eql) == NULL); + UPB_ASSERT_DEBUGVAR(val.ctype == t->ctype); + + t->count++; + mainpos_e = getentry_mutable(t, hash); + our_e = mainpos_e; + + if (upb_tabent_isempty(mainpos_e)) { + /* Our main position is empty; use it. */ + our_e->next = NULL; + } else { + /* Collision. */ + upb_tabent *new_e = emptyent(t); + /* Head of collider's chain. */ + upb_tabent *chain = getentry_mutable(t, hashfunc(mainpos_e->key)); + if (chain == mainpos_e) { + /* Existing ent is in its main posisiton (it has the same hash as us, and + * is the head of our chain). Insert to new ent and append to this chain. */ + new_e->next = mainpos_e->next; + mainpos_e->next = new_e; + our_e = new_e; + } else { + /* Existing ent is not in its main position (it is a node in some other + * chain). This implies that no existing ent in the table has our hash. + * Evict it (updating its chain) and use its ent for head of our chain. */ + *new_e = *mainpos_e; /* copies next. */ + while (chain->next != mainpos_e) { + chain = (upb_tabent*)chain->next; + UPB_ASSERT(chain); + } + chain->next = new_e; + our_e = mainpos_e; + our_e->next = NULL; + } + } + our_e->key = tabkey; + our_e->val.val = val.val; + UPB_ASSERT(findentry(t, key, hash, eql) == our_e); +} + +static bool rm(upb_table *t, lookupkey_t key, upb_value *val, + upb_tabkey *removed, uint32_t hash, eqlfunc_t *eql) { + upb_tabent *chain = getentry_mutable(t, hash); + if (upb_tabent_isempty(chain)) return false; + if (eql(chain->key, key)) { + /* Element to remove is at the head of its chain. */ + t->count--; + if (val) _upb_value_setval(val, chain->val.val, t->ctype); + if (removed) *removed = chain->key; + if (chain->next) { + upb_tabent *move = (upb_tabent*)chain->next; + *chain = *move; + move->key = 0; /* Make the slot empty. */ + } else { + chain->key = 0; /* Make the slot empty. */ + } + return true; + } else { + /* Element to remove is either in a non-head position or not in the + * table. */ + while (chain->next && !eql(chain->next->key, key)) { + chain = (upb_tabent*)chain->next; + } + if (chain->next) { + /* Found element to remove. */ + upb_tabent *rm = (upb_tabent*)chain->next; + t->count--; + if (val) _upb_value_setval(val, chain->next->val.val, t->ctype); + if (removed) *removed = rm->key; + rm->key = 0; /* Make the slot empty. */ + chain->next = rm->next; + return true; + } else { + /* Element to remove is not in the table. */ + return false; + } + } +} + +static size_t next(const upb_table *t, size_t i) { + do { + if (++i >= upb_table_size(t)) + return SIZE_MAX; + } while(upb_tabent_isempty(&t->entries[i])); + + return i; +} + +static size_t begin(const upb_table *t) { + return next(t, -1); +} + + +/* upb_strtable ***************************************************************/ + +/* A simple "subclass" of upb_table that only adds a hash function for strings. */ + +static upb_tabkey strcopy(lookupkey_t k2, upb_alloc *a) { + uint32_t len = (uint32_t) k2.str.len; + char *str = upb_malloc(a, k2.str.len + sizeof(uint32_t) + 1); + if (str == NULL) return 0; + memcpy(str, &len, sizeof(uint32_t)); + memcpy(str + sizeof(uint32_t), k2.str.str, k2.str.len); + str[sizeof(uint32_t) + k2.str.len] = '\0'; + return (uintptr_t)str; +} + +static uint32_t strhash(upb_tabkey key) { + uint32_t len; + char *str = upb_tabstr(key, &len); + return upb_murmur_hash2(str, len, 0); +} + +static bool streql(upb_tabkey k1, lookupkey_t k2) { + uint32_t len; + char *str = upb_tabstr(k1, &len); + return len == k2.str.len && memcmp(str, k2.str.str, len) == 0; +} + +bool upb_strtable_init2(upb_strtable *t, upb_ctype_t ctype, upb_alloc *a) { + return init(&t->t, ctype, 2, a); +} + +void upb_strtable_uninit2(upb_strtable *t, upb_alloc *a) { + size_t i; + for (i = 0; i < upb_table_size(&t->t); i++) + upb_free(a, (void*)t->t.entries[i].key); + uninit(&t->t, a); +} + +bool upb_strtable_resize(upb_strtable *t, size_t size_lg2, upb_alloc *a) { + upb_strtable new_table; + upb_strtable_iter i; + + upb_check_alloc(&t->t, a); + + if (!init(&new_table.t, t->t.ctype, size_lg2, a)) + return false; + upb_strtable_begin(&i, t); + for ( ; !upb_strtable_done(&i); upb_strtable_next(&i)) { + upb_strtable_insert3( + &new_table, + upb_strtable_iter_key(&i), + upb_strtable_iter_keylength(&i), + upb_strtable_iter_value(&i), + a); + } + upb_strtable_uninit2(t, a); + *t = new_table; + return true; +} + +bool upb_strtable_insert3(upb_strtable *t, const char *k, size_t len, + upb_value v, upb_alloc *a) { + lookupkey_t key; + upb_tabkey tabkey; + uint32_t hash; + + upb_check_alloc(&t->t, a); + + if (isfull(&t->t)) { + /* Need to resize. New table of double the size, add old elements to it. */ + if (!upb_strtable_resize(t, t->t.size_lg2 + 1, a)) { + return false; + } + } + + key = strkey2(k, len); + tabkey = strcopy(key, a); + if (tabkey == 0) return false; + + hash = upb_murmur_hash2(key.str.str, key.str.len, 0); + insert(&t->t, key, tabkey, v, hash, &strhash, &streql); + return true; +} + +bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len, + upb_value *v) { + uint32_t hash = upb_murmur_hash2(key, len, 0); + return lookup(&t->t, strkey2(key, len), v, hash, &streql); +} + +bool upb_strtable_remove3(upb_strtable *t, const char *key, size_t len, + upb_value *val, upb_alloc *alloc) { + uint32_t hash = upb_murmur_hash2(key, len, 0); + upb_tabkey tabkey; + if (rm(&t->t, strkey2(key, len), val, &tabkey, hash, &streql)) { + upb_free(alloc, (void*)tabkey); + return true; + } else { + return false; + } +} + +/* Iteration */ + +static const upb_tabent *str_tabent(const upb_strtable_iter *i) { + return &i->t->t.entries[i->index]; +} + +void upb_strtable_begin(upb_strtable_iter *i, const upb_strtable *t) { + i->t = t; + i->index = begin(&t->t); +} + +void upb_strtable_next(upb_strtable_iter *i) { + i->index = next(&i->t->t, i->index); +} + +bool upb_strtable_done(const upb_strtable_iter *i) { + if (!i->t) return true; + return i->index >= upb_table_size(&i->t->t) || + upb_tabent_isempty(str_tabent(i)); +} + +const char *upb_strtable_iter_key(const upb_strtable_iter *i) { + UPB_ASSERT(!upb_strtable_done(i)); + return upb_tabstr(str_tabent(i)->key, NULL); +} + +size_t upb_strtable_iter_keylength(const upb_strtable_iter *i) { + uint32_t len; + UPB_ASSERT(!upb_strtable_done(i)); + upb_tabstr(str_tabent(i)->key, &len); + return len; +} + +upb_value upb_strtable_iter_value(const upb_strtable_iter *i) { + UPB_ASSERT(!upb_strtable_done(i)); + return _upb_value_val(str_tabent(i)->val.val, i->t->t.ctype); +} + +void upb_strtable_iter_setdone(upb_strtable_iter *i) { + i->t = NULL; + i->index = SIZE_MAX; +} + +bool upb_strtable_iter_isequal(const upb_strtable_iter *i1, + const upb_strtable_iter *i2) { + if (upb_strtable_done(i1) && upb_strtable_done(i2)) + return true; + return i1->t == i2->t && i1->index == i2->index; +} + + +/* upb_inttable ***************************************************************/ + +/* For inttables we use a hybrid structure where small keys are kept in an + * array and large keys are put in the hash table. */ + +static uint32_t inthash(upb_tabkey key) { return upb_inthash(key); } + +static bool inteql(upb_tabkey k1, lookupkey_t k2) { + return k1 == k2.num; +} + +static upb_tabval *mutable_array(upb_inttable *t) { + return (upb_tabval*)t->array; +} + +static upb_tabval *inttable_val(upb_inttable *t, uintptr_t key) { + if (key < t->array_size) { + return upb_arrhas(t->array[key]) ? &(mutable_array(t)[key]) : NULL; + } else { + upb_tabent *e = + findentry_mutable(&t->t, intkey(key), upb_inthash(key), &inteql); + return e ? &e->val : NULL; + } +} + +static const upb_tabval *inttable_val_const(const upb_inttable *t, + uintptr_t key) { + return inttable_val((upb_inttable*)t, key); +} + +size_t upb_inttable_count(const upb_inttable *t) { + return t->t.count + t->array_count; +} + +static void check(upb_inttable *t) { + UPB_UNUSED(t); +#if defined(UPB_DEBUG_TABLE) && !defined(NDEBUG) + { + /* This check is very expensive (makes inserts/deletes O(N)). */ + size_t count = 0; + upb_inttable_iter i; + upb_inttable_begin(&i, t); + for(; !upb_inttable_done(&i); upb_inttable_next(&i), count++) { + UPB_ASSERT(upb_inttable_lookup(t, upb_inttable_iter_key(&i), NULL)); + } + UPB_ASSERT(count == upb_inttable_count(t)); + } +#endif +} + +bool upb_inttable_sizedinit(upb_inttable *t, upb_ctype_t ctype, + size_t asize, int hsize_lg2, upb_alloc *a) { + size_t array_bytes; + + if (!init(&t->t, ctype, hsize_lg2, a)) return false; + /* Always make the array part at least 1 long, so that we know key 0 + * won't be in the hash part, which simplifies things. */ + t->array_size = UPB_MAX(1, asize); + t->array_count = 0; + array_bytes = t->array_size * sizeof(upb_value); + t->array = upb_malloc(a, array_bytes); + if (!t->array) { + uninit(&t->t, a); + return false; + } + memset(mutable_array(t), 0xff, array_bytes); + check(t); + return true; +} + +bool upb_inttable_init2(upb_inttable *t, upb_ctype_t ctype, upb_alloc *a) { + return upb_inttable_sizedinit(t, ctype, 0, 4, a); +} + +void upb_inttable_uninit2(upb_inttable *t, upb_alloc *a) { + uninit(&t->t, a); + upb_free(a, mutable_array(t)); +} + +bool upb_inttable_insert2(upb_inttable *t, uintptr_t key, upb_value val, + upb_alloc *a) { + upb_tabval tabval; + tabval.val = val.val; + UPB_ASSERT(upb_arrhas(tabval)); /* This will reject (uint64_t)-1. Fix this. */ + + upb_check_alloc(&t->t, a); + + if (key < t->array_size) { + UPB_ASSERT(!upb_arrhas(t->array[key])); + t->array_count++; + mutable_array(t)[key].val = val.val; + } else { + if (isfull(&t->t)) { + /* Need to resize the hash part, but we re-use the array part. */ + size_t i; + upb_table new_table; + + if (!init(&new_table, t->t.ctype, t->t.size_lg2 + 1, a)) { + return false; + } + + for (i = begin(&t->t); i < upb_table_size(&t->t); i = next(&t->t, i)) { + const upb_tabent *e = &t->t.entries[i]; + uint32_t hash; + upb_value v; + + _upb_value_setval(&v, e->val.val, t->t.ctype); + hash = upb_inthash(e->key); + insert(&new_table, intkey(e->key), e->key, v, hash, &inthash, &inteql); + } + + UPB_ASSERT(t->t.count == new_table.count); + + uninit(&t->t, a); + t->t = new_table; + } + insert(&t->t, intkey(key), key, val, upb_inthash(key), &inthash, &inteql); + } + check(t); + return true; +} + +bool upb_inttable_lookup(const upb_inttable *t, uintptr_t key, upb_value *v) { + const upb_tabval *table_v = inttable_val_const(t, key); + if (!table_v) return false; + if (v) _upb_value_setval(v, table_v->val, t->t.ctype); + return true; +} + +bool upb_inttable_replace(upb_inttable *t, uintptr_t key, upb_value val) { + upb_tabval *table_v = inttable_val(t, key); + if (!table_v) return false; + table_v->val = val.val; + return true; +} + +bool upb_inttable_remove(upb_inttable *t, uintptr_t key, upb_value *val) { + bool success; + if (key < t->array_size) { + if (upb_arrhas(t->array[key])) { + upb_tabval empty = UPB_TABVALUE_EMPTY_INIT; + t->array_count--; + if (val) { + _upb_value_setval(val, t->array[key].val, t->t.ctype); + } + mutable_array(t)[key] = empty; + success = true; + } else { + success = false; + } + } else { + success = rm(&t->t, intkey(key), val, NULL, upb_inthash(key), &inteql); + } + check(t); + return success; +} + +bool upb_inttable_push2(upb_inttable *t, upb_value val, upb_alloc *a) { + upb_check_alloc(&t->t, a); + return upb_inttable_insert2(t, upb_inttable_count(t), val, a); +} + +upb_value upb_inttable_pop(upb_inttable *t) { + upb_value val; + bool ok = upb_inttable_remove(t, upb_inttable_count(t) - 1, &val); + UPB_ASSERT(ok); + return val; +} + +bool upb_inttable_insertptr2(upb_inttable *t, const void *key, upb_value val, + upb_alloc *a) { + upb_check_alloc(&t->t, a); + return upb_inttable_insert2(t, (uintptr_t)key, val, a); +} + +bool upb_inttable_lookupptr(const upb_inttable *t, const void *key, + upb_value *v) { + return upb_inttable_lookup(t, (uintptr_t)key, v); +} + +bool upb_inttable_removeptr(upb_inttable *t, const void *key, upb_value *val) { + return upb_inttable_remove(t, (uintptr_t)key, val); +} + +void upb_inttable_compact2(upb_inttable *t, upb_alloc *a) { + /* A power-of-two histogram of the table keys. */ + size_t counts[UPB_MAXARRSIZE + 1] = {0}; + + /* The max key in each bucket. */ + uintptr_t max[UPB_MAXARRSIZE + 1] = {0}; + + upb_inttable_iter i; + size_t arr_count; + int size_lg2; + upb_inttable new_t; + + upb_check_alloc(&t->t, a); + + upb_inttable_begin(&i, t); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + uintptr_t key = upb_inttable_iter_key(&i); + int bucket = log2ceil(key); + max[bucket] = UPB_MAX(max[bucket], key); + counts[bucket]++; + } + + /* Find the largest power of two that satisfies the MIN_DENSITY + * definition (while actually having some keys). */ + arr_count = upb_inttable_count(t); + + for (size_lg2 = ARRAY_SIZE(counts) - 1; size_lg2 > 0; size_lg2--) { + if (counts[size_lg2] == 0) { + /* We can halve again without losing any entries. */ + continue; + } else if (arr_count >= (1 << size_lg2) * MIN_DENSITY) { + break; + } + + arr_count -= counts[size_lg2]; + } + + UPB_ASSERT(arr_count <= upb_inttable_count(t)); + + { + /* Insert all elements into new, perfectly-sized table. */ + size_t arr_size = max[size_lg2] + 1; /* +1 so arr[max] will fit. */ + size_t hash_count = upb_inttable_count(t) - arr_count; + size_t hash_size = hash_count ? (hash_count / MAX_LOAD) + 1 : 0; + int hashsize_lg2 = log2ceil(hash_size); + + upb_inttable_sizedinit(&new_t, t->t.ctype, arr_size, hashsize_lg2, a); + upb_inttable_begin(&i, t); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + uintptr_t k = upb_inttable_iter_key(&i); + upb_inttable_insert2(&new_t, k, upb_inttable_iter_value(&i), a); + } + UPB_ASSERT(new_t.array_size == arr_size); + UPB_ASSERT(new_t.t.size_lg2 == hashsize_lg2); + } + upb_inttable_uninit2(t, a); + *t = new_t; +} + +/* Iteration. */ + +static const upb_tabent *int_tabent(const upb_inttable_iter *i) { + UPB_ASSERT(!i->array_part); + return &i->t->t.entries[i->index]; +} + +static upb_tabval int_arrent(const upb_inttable_iter *i) { + UPB_ASSERT(i->array_part); + return i->t->array[i->index]; +} + +void upb_inttable_begin(upb_inttable_iter *i, const upb_inttable *t) { + i->t = t; + i->index = -1; + i->array_part = true; + upb_inttable_next(i); +} + +void upb_inttable_next(upb_inttable_iter *iter) { + const upb_inttable *t = iter->t; + if (iter->array_part) { + while (++iter->index < t->array_size) { + if (upb_arrhas(int_arrent(iter))) { + return; + } + } + iter->array_part = false; + iter->index = begin(&t->t); + } else { + iter->index = next(&t->t, iter->index); + } +} + +bool upb_inttable_done(const upb_inttable_iter *i) { + if (!i->t) return true; + if (i->array_part) { + return i->index >= i->t->array_size || + !upb_arrhas(int_arrent(i)); + } else { + return i->index >= upb_table_size(&i->t->t) || + upb_tabent_isempty(int_tabent(i)); + } +} + +uintptr_t upb_inttable_iter_key(const upb_inttable_iter *i) { + UPB_ASSERT(!upb_inttable_done(i)); + return i->array_part ? i->index : int_tabent(i)->key; +} + +upb_value upb_inttable_iter_value(const upb_inttable_iter *i) { + UPB_ASSERT(!upb_inttable_done(i)); + return _upb_value_val( + i->array_part ? i->t->array[i->index].val : int_tabent(i)->val.val, + i->t->t.ctype); +} + +void upb_inttable_iter_setdone(upb_inttable_iter *i) { + i->t = NULL; + i->index = SIZE_MAX; + i->array_part = false; +} + +bool upb_inttable_iter_isequal(const upb_inttable_iter *i1, + const upb_inttable_iter *i2) { + if (upb_inttable_done(i1) && upb_inttable_done(i2)) + return true; + return i1->t == i2->t && i1->index == i2->index && + i1->array_part == i2->array_part; +} + +#if defined(UPB_UNALIGNED_READS_OK) || defined(__s390x__) +/* ----------------------------------------------------------------------------- + * MurmurHash2, by Austin Appleby (released as public domain). + * Reformatted and C99-ified by Joshua Haberman. + * Note - This code makes a few assumptions about how your machine behaves - + * 1. We can read a 4-byte value from any address without crashing + * 2. sizeof(int) == 4 (in upb this limitation is removed by using uint32_t + * And it has a few limitations - + * 1. It will not work incrementally. + * 2. It will not produce the same results on little-endian and big-endian + * machines. */ +uint32_t upb_murmur_hash2(const void *key, size_t len, uint32_t seed) { + /* 'm' and 'r' are mixing constants generated offline. + * They're not really 'magic', they just happen to work well. */ + const uint32_t m = 0x5bd1e995; + const int32_t r = 24; + + /* Initialize the hash to a 'random' value */ + uint32_t h = seed ^ len; + + /* Mix 4 bytes at a time into the hash */ + const uint8_t * data = (const uint8_t *)key; + while(len >= 4) { + uint32_t k = *(uint32_t *)data; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + /* Handle the last few bytes of the input array */ + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + + /* Do a few final mixes of the hash to ensure the last few + * bytes are well-incorporated. */ + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +#else /* !UPB_UNALIGNED_READS_OK */ + +/* ----------------------------------------------------------------------------- + * MurmurHashAligned2, by Austin Appleby + * Same algorithm as MurmurHash2, but only does aligned reads - should be safer + * on certain platforms. + * Performance will be lower than MurmurHash2 */ + +#define MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; } + +uint32_t upb_murmur_hash2(const void * key, size_t len, uint32_t seed) { + const uint32_t m = 0x5bd1e995; + const int32_t r = 24; + const uint8_t * data = (const uint8_t *)key; + uint32_t h = (uint32_t)(seed ^ len); + uint8_t align = (uintptr_t)data & 3; + + if(align && (len >= 4)) { + /* Pre-load the temp registers */ + uint32_t t = 0, d = 0; + int32_t sl; + int32_t sr; + + switch(align) { + case 1: t |= data[2] << 16; + case 2: t |= data[1] << 8; + case 3: t |= data[0]; + } + + t <<= (8 * align); + + data += 4-align; + len -= 4-align; + + sl = 8 * (4-align); + sr = 8 * align; + + /* Mix */ + + while(len >= 4) { + uint32_t k; + + d = *(uint32_t *)data; + t = (t >> sr) | (d << sl); + + k = t; + + MIX(h,k,m); + + t = d; + + data += 4; + len -= 4; + } + + /* Handle leftover data in temp registers */ + + d = 0; + + if(len >= align) { + uint32_t k; + + switch(align) { + case 3: d |= data[2] << 16; + case 2: d |= data[1] << 8; + case 1: d |= data[0]; + } + + k = (t >> sr) | (d << sl); + MIX(h,k,m); + + data += align; + len -= align; + + /* ---------- + * Handle tail bytes */ + + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + } else { + switch(len) { + case 3: d |= data[2] << 16; + case 2: d |= data[1] << 8; + case 1: d |= data[0]; + case 0: h ^= (t >> sr) | (d << sl); h *= m; + } + } + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; + } else { + while(len >= 4) { + uint32_t k = *(uint32_t *)data; + + MIX(h,k,m); + + data += 4; + len -= 4; + } + + /* ---------- + * Handle tail bytes */ + + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; + } +} +#undef MIX + +#endif /* UPB_UNALIGNED_READS_OK */ diff --git a/upb/table.int.h b/upb/table.int.h new file mode 100644 index 00000000000..23b0b2f2213 --- /dev/null +++ b/upb/table.int.h @@ -0,0 +1,507 @@ +/* +** upb_table +** +** This header is INTERNAL-ONLY! Its interfaces are not public or stable! +** This file defines very fast int->upb_value (inttable) and string->upb_value +** (strtable) hash tables. +** +** The table uses chained scatter with Brent's variation (inspired by the Lua +** implementation of hash tables). The hash function for strings is Austin +** Appleby's "MurmurHash." +** +** The inttable uses uintptr_t as its key, which guarantees it can be used to +** store pointers or integers of at least 32 bits (upb isn't really useful on +** systems where sizeof(void*) < 4). +** +** The table must be homogenous (all values of the same type). In debug +** mode, we check this on insert and lookup. +*/ + +#ifndef UPB_TABLE_H_ +#define UPB_TABLE_H_ + +#include +#include +#include "upb/upb.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + + +/* upb_value ******************************************************************/ + +/* A tagged union (stored untagged inside the table) so that we can check that + * clients calling table accessors are correctly typed without having to have + * an explosion of accessors. */ +typedef enum { + UPB_CTYPE_INT32 = 1, + UPB_CTYPE_INT64 = 2, + UPB_CTYPE_UINT32 = 3, + UPB_CTYPE_UINT64 = 4, + UPB_CTYPE_BOOL = 5, + UPB_CTYPE_CSTR = 6, + UPB_CTYPE_PTR = 7, + UPB_CTYPE_CONSTPTR = 8, + UPB_CTYPE_FPTR = 9, + UPB_CTYPE_FLOAT = 10, + UPB_CTYPE_DOUBLE = 11 +} upb_ctype_t; + +typedef struct { + uint64_t val; +#ifndef NDEBUG + /* In debug mode we carry the value type around also so we can check accesses + * to be sure the right member is being read. */ + upb_ctype_t ctype; +#endif +} upb_value; + +#ifdef NDEBUG +#define SET_TYPE(dest, val) UPB_UNUSED(val) +#else +#define SET_TYPE(dest, val) dest = val +#endif + +/* Like strdup(), which isn't always available since it's not ANSI C. */ +char *upb_strdup(const char *s, upb_alloc *a); +/* Variant that works with a length-delimited rather than NULL-delimited string, + * as supported by strtable. */ +char *upb_strdup2(const char *s, size_t len, upb_alloc *a); + +UPB_INLINE char *upb_gstrdup(const char *s) { + return upb_strdup(s, &upb_alloc_global); +} + +UPB_INLINE void _upb_value_setval(upb_value *v, uint64_t val, + upb_ctype_t ctype) { + v->val = val; + SET_TYPE(v->ctype, ctype); +} + +UPB_INLINE upb_value _upb_value_val(uint64_t val, upb_ctype_t ctype) { + upb_value ret; + _upb_value_setval(&ret, val, ctype); + return ret; +} + +/* For each value ctype, define the following set of functions: + * + * // Get/set an int32 from a upb_value. + * int32_t upb_value_getint32(upb_value val); + * void upb_value_setint32(upb_value *val, int32_t cval); + * + * // Construct a new upb_value from an int32. + * upb_value upb_value_int32(int32_t val); */ +#define FUNCS(name, membername, type_t, converter, proto_type) \ + UPB_INLINE void upb_value_set ## name(upb_value *val, type_t cval) { \ + val->val = (converter)cval; \ + SET_TYPE(val->ctype, proto_type); \ + } \ + UPB_INLINE upb_value upb_value_ ## name(type_t val) { \ + upb_value ret; \ + upb_value_set ## name(&ret, val); \ + return ret; \ + } \ + UPB_INLINE type_t upb_value_get ## name(upb_value val) { \ + UPB_ASSERT_DEBUGVAR(val.ctype == proto_type); \ + return (type_t)(converter)val.val; \ + } + +FUNCS(int32, int32, int32_t, int32_t, UPB_CTYPE_INT32) +FUNCS(int64, int64, int64_t, int64_t, UPB_CTYPE_INT64) +FUNCS(uint32, uint32, uint32_t, uint32_t, UPB_CTYPE_UINT32) +FUNCS(uint64, uint64, uint64_t, uint64_t, UPB_CTYPE_UINT64) +FUNCS(bool, _bool, bool, bool, UPB_CTYPE_BOOL) +FUNCS(cstr, cstr, char*, uintptr_t, UPB_CTYPE_CSTR) +FUNCS(ptr, ptr, void*, uintptr_t, UPB_CTYPE_PTR) +FUNCS(constptr, constptr, const void*, uintptr_t, UPB_CTYPE_CONSTPTR) +FUNCS(fptr, fptr, upb_func*, uintptr_t, UPB_CTYPE_FPTR) + +#undef FUNCS + +UPB_INLINE void upb_value_setfloat(upb_value *val, float cval) { + memcpy(&val->val, &cval, sizeof(cval)); + SET_TYPE(val->ctype, UPB_CTYPE_FLOAT); +} + +UPB_INLINE void upb_value_setdouble(upb_value *val, double cval) { + memcpy(&val->val, &cval, sizeof(cval)); + SET_TYPE(val->ctype, UPB_CTYPE_DOUBLE); +} + +UPB_INLINE upb_value upb_value_float(float cval) { + upb_value ret; + upb_value_setfloat(&ret, cval); + return ret; +} + +UPB_INLINE upb_value upb_value_double(double cval) { + upb_value ret; + upb_value_setdouble(&ret, cval); + return ret; +} + +#undef SET_TYPE + + +/* upb_tabkey *****************************************************************/ + +/* Either: + * 1. an actual integer key, or + * 2. a pointer to a string prefixed by its uint32_t length, owned by us. + * + * ...depending on whether this is a string table or an int table. We would + * make this a union of those two types, but C89 doesn't support statically + * initializing a non-first union member. */ +typedef uintptr_t upb_tabkey; + +UPB_INLINE char *upb_tabstr(upb_tabkey key, uint32_t *len) { + char* mem = (char*)key; + if (len) memcpy(len, mem, sizeof(*len)); + return mem + sizeof(*len); +} + + +/* upb_tabval *****************************************************************/ + +typedef struct { + uint64_t val; +} upb_tabval; + +#define UPB_TABVALUE_EMPTY_INIT {-1} + + +/* upb_table ******************************************************************/ + +typedef struct _upb_tabent { + upb_tabkey key; + upb_tabval val; + + /* Internal chaining. This is const so we can create static initializers for + * tables. We cast away const sometimes, but *only* when the containing + * upb_table is known to be non-const. This requires a bit of care, but + * the subtlety is confined to table.c. */ + const struct _upb_tabent *next; +} upb_tabent; + +typedef struct { + size_t count; /* Number of entries in the hash part. */ + size_t mask; /* Mask to turn hash value -> bucket. */ + upb_ctype_t ctype; /* Type of all values. */ + uint8_t size_lg2; /* Size of the hashtable part is 2^size_lg2 entries. */ + + /* Hash table entries. + * Making this const isn't entirely accurate; what we really want is for it to + * have the same const-ness as the table it's inside. But there's no way to + * declare that in C. So we have to make it const so that we can statically + * initialize const hash tables. Then we cast away const when we have to. + */ + const upb_tabent *entries; + +#ifndef NDEBUG + /* This table's allocator. We make the user pass it in to every relevant + * function and only use this to check it in debug mode. We do this solely + * to keep upb_table as small as possible. This might seem slightly paranoid + * but the plan is to use upb_table for all map fields and extension sets in + * a forthcoming message representation, so there could be a lot of these. + * If this turns out to be too annoying later, we can change it (since this + * is an internal-only header file). */ + upb_alloc *alloc; +#endif +} upb_table; + +typedef struct { + upb_table t; +} upb_strtable; + +typedef struct { + upb_table t; /* For entries that don't fit in the array part. */ + const upb_tabval *array; /* Array part of the table. See const note above. */ + size_t array_size; /* Array part size. */ + size_t array_count; /* Array part number of elements. */ +} upb_inttable; + +#define UPB_INTTABLE_INIT(count, mask, ctype, size_lg2, ent, a, asize, acount) \ + {UPB_TABLE_INIT(count, mask, ctype, size_lg2, ent), a, asize, acount} + +#define UPB_EMPTY_INTTABLE_INIT(ctype) \ + UPB_INTTABLE_INIT(0, 0, ctype, 0, NULL, NULL, 0, 0) + +#define UPB_ARRAY_EMPTYENT -1 + +UPB_INLINE size_t upb_table_size(const upb_table *t) { + if (t->size_lg2 == 0) + return 0; + else + return 1 << t->size_lg2; +} + +/* Internal-only functions, in .h file only out of necessity. */ +UPB_INLINE bool upb_tabent_isempty(const upb_tabent *e) { + return e->key == 0; +} + +/* Used by some of the unit tests for generic hashing functionality. */ +uint32_t upb_murmur_hash2(const void * key, size_t len, uint32_t seed); + +UPB_INLINE uintptr_t upb_intkey(uintptr_t key) { + return key; +} + +UPB_INLINE uint32_t upb_inthash(uintptr_t key) { + return (uint32_t)key; +} + +static const upb_tabent *upb_getentry(const upb_table *t, uint32_t hash) { + return t->entries + (hash & t->mask); +} + +UPB_INLINE bool upb_arrhas(upb_tabval key) { + return key.val != (uint64_t)-1; +} + +/* Initialize and uninitialize a table, respectively. If memory allocation + * failed, false is returned that the table is uninitialized. */ +bool upb_inttable_init2(upb_inttable *table, upb_ctype_t ctype, upb_alloc *a); +bool upb_strtable_init2(upb_strtable *table, upb_ctype_t ctype, upb_alloc *a); +void upb_inttable_uninit2(upb_inttable *table, upb_alloc *a); +void upb_strtable_uninit2(upb_strtable *table, upb_alloc *a); + +UPB_INLINE bool upb_inttable_init(upb_inttable *table, upb_ctype_t ctype) { + return upb_inttable_init2(table, ctype, &upb_alloc_global); +} + +UPB_INLINE bool upb_strtable_init(upb_strtable *table, upb_ctype_t ctype) { + return upb_strtable_init2(table, ctype, &upb_alloc_global); +} + +UPB_INLINE void upb_inttable_uninit(upb_inttable *table) { + upb_inttable_uninit2(table, &upb_alloc_global); +} + +UPB_INLINE void upb_strtable_uninit(upb_strtable *table) { + upb_strtable_uninit2(table, &upb_alloc_global); +} + +/* Returns the number of values in the table. */ +size_t upb_inttable_count(const upb_inttable *t); +UPB_INLINE size_t upb_strtable_count(const upb_strtable *t) { + return t->t.count; +} + +void upb_inttable_packedsize(const upb_inttable *t, size_t *size); +void upb_strtable_packedsize(const upb_strtable *t, size_t *size); +upb_inttable *upb_inttable_pack(const upb_inttable *t, void *p, size_t *ofs, + size_t size); +upb_strtable *upb_strtable_pack(const upb_strtable *t, void *p, size_t *ofs, + size_t size); + +/* Inserts the given key into the hashtable with the given value. The key must + * not already exist in the hash table. For string tables, the key must be + * NULL-terminated, and the table will make an internal copy of the key. + * Inttables must not insert a value of UINTPTR_MAX. + * + * If a table resize was required but memory allocation failed, false is + * returned and the table is unchanged. */ +bool upb_inttable_insert2(upb_inttable *t, uintptr_t key, upb_value val, + upb_alloc *a); +bool upb_strtable_insert3(upb_strtable *t, const char *key, size_t len, + upb_value val, upb_alloc *a); + +UPB_INLINE bool upb_inttable_insert(upb_inttable *t, uintptr_t key, + upb_value val) { + return upb_inttable_insert2(t, key, val, &upb_alloc_global); +} + +UPB_INLINE bool upb_strtable_insert2(upb_strtable *t, const char *key, + size_t len, upb_value val) { + return upb_strtable_insert3(t, key, len, val, &upb_alloc_global); +} + +/* For NULL-terminated strings. */ +UPB_INLINE bool upb_strtable_insert(upb_strtable *t, const char *key, + upb_value val) { + return upb_strtable_insert2(t, key, strlen(key), val); +} + +/* Looks up key in this table, returning "true" if the key was found. + * If v is non-NULL, copies the value for this key into *v. */ +bool upb_inttable_lookup(const upb_inttable *t, uintptr_t key, upb_value *v); +bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len, + upb_value *v); + +/* For NULL-terminated strings. */ +UPB_INLINE bool upb_strtable_lookup(const upb_strtable *t, const char *key, + upb_value *v) { + return upb_strtable_lookup2(t, key, strlen(key), v); +} + +/* Removes an item from the table. Returns true if the remove was successful, + * and stores the removed item in *val if non-NULL. */ +bool upb_inttable_remove(upb_inttable *t, uintptr_t key, upb_value *val); +bool upb_strtable_remove3(upb_strtable *t, const char *key, size_t len, + upb_value *val, upb_alloc *alloc); + +UPB_INLINE bool upb_strtable_remove2(upb_strtable *t, const char *key, + size_t len, upb_value *val) { + return upb_strtable_remove3(t, key, len, val, &upb_alloc_global); +} + +/* For NULL-terminated strings. */ +UPB_INLINE bool upb_strtable_remove(upb_strtable *t, const char *key, + upb_value *v) { + return upb_strtable_remove2(t, key, strlen(key), v); +} + +/* Updates an existing entry in an inttable. If the entry does not exist, + * returns false and does nothing. Unlike insert/remove, this does not + * invalidate iterators. */ +bool upb_inttable_replace(upb_inttable *t, uintptr_t key, upb_value val); + +/* Handy routines for treating an inttable like a stack. May not be mixed with + * other insert/remove calls. */ +bool upb_inttable_push2(upb_inttable *t, upb_value val, upb_alloc *a); +upb_value upb_inttable_pop(upb_inttable *t); + +UPB_INLINE bool upb_inttable_push(upb_inttable *t, upb_value val) { + return upb_inttable_push2(t, val, &upb_alloc_global); +} + +/* Convenience routines for inttables with pointer keys. */ +bool upb_inttable_insertptr2(upb_inttable *t, const void *key, upb_value val, + upb_alloc *a); +bool upb_inttable_removeptr(upb_inttable *t, const void *key, upb_value *val); +bool upb_inttable_lookupptr( + const upb_inttable *t, const void *key, upb_value *val); + +UPB_INLINE bool upb_inttable_insertptr(upb_inttable *t, const void *key, + upb_value val) { + return upb_inttable_insertptr2(t, key, val, &upb_alloc_global); +} + +/* Optimizes the table for the current set of entries, for both memory use and + * lookup time. Client should call this after all entries have been inserted; + * inserting more entries is legal, but will likely require a table resize. */ +void upb_inttable_compact2(upb_inttable *t, upb_alloc *a); + +UPB_INLINE void upb_inttable_compact(upb_inttable *t) { + upb_inttable_compact2(t, &upb_alloc_global); +} + +/* A special-case inlinable version of the lookup routine for 32-bit + * integers. */ +UPB_INLINE bool upb_inttable_lookup32(const upb_inttable *t, uint32_t key, + upb_value *v) { + *v = upb_value_int32(0); /* Silence compiler warnings. */ + if (key < t->array_size) { + upb_tabval arrval = t->array[key]; + if (upb_arrhas(arrval)) { + _upb_value_setval(v, arrval.val, t->t.ctype); + return true; + } else { + return false; + } + } else { + const upb_tabent *e; + if (t->t.entries == NULL) return false; + for (e = upb_getentry(&t->t, upb_inthash(key)); true; e = e->next) { + if ((uint32_t)e->key == key) { + _upb_value_setval(v, e->val.val, t->t.ctype); + return true; + } + if (e->next == NULL) return false; + } + } +} + +/* Exposed for testing only. */ +bool upb_strtable_resize(upb_strtable *t, size_t size_lg2, upb_alloc *a); + +/* Iterators ******************************************************************/ + +/* Iterators for int and string tables. We are subject to some kind of unusual + * design constraints: + * + * For high-level languages: + * - we must be able to guarantee that we don't crash or corrupt memory even if + * the program accesses an invalidated iterator. + * + * For C++11 range-based for: + * - iterators must be copyable + * - iterators must be comparable + * - it must be possible to construct an "end" value. + * + * Iteration order is undefined. + * + * Modifying the table invalidates iterators. upb_{str,int}table_done() is + * guaranteed to work even on an invalidated iterator, as long as the table it + * is iterating over has not been freed. Calling next() or accessing data from + * an invalidated iterator yields unspecified elements from the table, but it is + * guaranteed not to crash and to return real table elements (except when done() + * is true). */ + + +/* upb_strtable_iter **********************************************************/ + +/* upb_strtable_iter i; + * upb_strtable_begin(&i, t); + * for(; !upb_strtable_done(&i); upb_strtable_next(&i)) { + * const char *key = upb_strtable_iter_key(&i); + * const upb_value val = upb_strtable_iter_value(&i); + * // ... + * } + */ + +typedef struct { + const upb_strtable *t; + size_t index; +} upb_strtable_iter; + +void upb_strtable_begin(upb_strtable_iter *i, const upb_strtable *t); +void upb_strtable_next(upb_strtable_iter *i); +bool upb_strtable_done(const upb_strtable_iter *i); +const char *upb_strtable_iter_key(const upb_strtable_iter *i); +size_t upb_strtable_iter_keylength(const upb_strtable_iter *i); +upb_value upb_strtable_iter_value(const upb_strtable_iter *i); +void upb_strtable_iter_setdone(upb_strtable_iter *i); +bool upb_strtable_iter_isequal(const upb_strtable_iter *i1, + const upb_strtable_iter *i2); + + +/* upb_inttable_iter **********************************************************/ + +/* upb_inttable_iter i; + * upb_inttable_begin(&i, t); + * for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + * uintptr_t key = upb_inttable_iter_key(&i); + * upb_value val = upb_inttable_iter_value(&i); + * // ... + * } + */ + +typedef struct { + const upb_inttable *t; + size_t index; + bool array_part; +} upb_inttable_iter; + +void upb_inttable_begin(upb_inttable_iter *i, const upb_inttable *t); +void upb_inttable_next(upb_inttable_iter *i); +bool upb_inttable_done(const upb_inttable_iter *i); +uintptr_t upb_inttable_iter_key(const upb_inttable_iter *i); +upb_value upb_inttable_iter_value(const upb_inttable_iter *i); +void upb_inttable_iter_setdone(upb_inttable_iter *i); +bool upb_inttable_iter_isequal(const upb_inttable_iter *i1, + const upb_inttable_iter *i2); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* UPB_TABLE_H_ */ diff --git a/upb/upb.c b/upb/upb.c new file mode 100644 index 00000000000..266ea7d7f98 --- /dev/null +++ b/upb/upb.c @@ -0,0 +1,261 @@ + +#include "upb/upb.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "upb/port_def.inc" + +/* Guarantee null-termination and provide ellipsis truncation. + * It may be tempting to "optimize" this by initializing these final + * four bytes up-front and then being careful never to overwrite them, + * this is safer and simpler. */ +static void nullz(upb_status *status) { + const char *ellipsis = "..."; + size_t len = strlen(ellipsis); + UPB_ASSERT(sizeof(status->msg) > len); + memcpy(status->msg + sizeof(status->msg) - len, ellipsis, len); +} + +/* upb_status *****************************************************************/ + +void upb_status_clear(upb_status *status) { + if (!status) return; + status->ok = true; + status->msg[0] = '\0'; +} + +bool upb_ok(const upb_status *status) { return status->ok; } + +const char *upb_status_errmsg(const upb_status *status) { return status->msg; } + +void upb_status_seterrmsg(upb_status *status, const char *msg) { + if (!status) return; + status->ok = false; + strncpy(status->msg, msg, sizeof(status->msg)); + nullz(status); +} + +void upb_status_seterrf(upb_status *status, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + upb_status_vseterrf(status, fmt, args); + va_end(args); +} + +void upb_status_vseterrf(upb_status *status, const char *fmt, va_list args) { + if (!status) return; + status->ok = false; + _upb_vsnprintf(status->msg, sizeof(status->msg), fmt, args); + nullz(status); +} + +/* upb_alloc ******************************************************************/ + +static void *upb_global_allocfunc(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size) { + UPB_UNUSED(alloc); + UPB_UNUSED(oldsize); + if (size == 0) { + free(ptr); + return NULL; + } else { + return realloc(ptr, size); + } +} + +upb_alloc upb_alloc_global = {&upb_global_allocfunc}; + +/* upb_arena ******************************************************************/ + +/* Be conservative and choose 16 in case anyone is using SSE. */ +static const size_t maxalign = 16; + +static size_t align_up_max(size_t size) { + return ((size + maxalign - 1) / maxalign) * maxalign; +} + +struct upb_arena { + /* We implement the allocator interface. + * This must be the first member of upb_arena! */ + upb_alloc alloc; + + /* Allocator to allocate arena blocks. We are responsible for freeing these + * when we are destroyed. */ + upb_alloc *block_alloc; + + size_t bytes_allocated; + size_t next_block_size; + size_t max_block_size; + + /* Linked list of blocks. Points to an arena_block, defined in env.c */ + void *block_head; + + /* Cleanup entries. Pointer to a cleanup_ent, defined in env.c */ + void *cleanup_head; +}; + +typedef struct mem_block { + struct mem_block *next; + size_t size; + size_t used; + bool owned; + /* Data follows. */ +} mem_block; + +typedef struct cleanup_ent { + struct cleanup_ent *next; + upb_cleanup_func *cleanup; + void *ud; +} cleanup_ent; + +static void upb_arena_addblock(upb_arena *a, void *ptr, size_t size, + bool owned) { + mem_block *block = ptr; + + block->next = a->block_head; + block->size = size; + block->used = align_up_max(sizeof(mem_block)); + block->owned = owned; + + a->block_head = block; + + /* TODO(haberman): ASAN poison. */ +} + +static mem_block *upb_arena_allocblock(upb_arena *a, size_t size) { + size_t block_size = UPB_MAX(size, a->next_block_size) + sizeof(mem_block); + mem_block *block = upb_malloc(a->block_alloc, block_size); + + if (!block) { + return NULL; + } + + upb_arena_addblock(a, block, block_size, true); + a->next_block_size = UPB_MIN(block_size * 2, a->max_block_size); + + return block; +} + +static void *upb_arena_doalloc(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size) { + upb_arena *a = (upb_arena*)alloc; /* upb_alloc is initial member. */ + mem_block *block = a->block_head; + void *ret; + + if (size == 0) { + return NULL; /* We are an arena, don't need individual frees. */ + } + + size = align_up_max(size); + + /* TODO(haberman): special-case if this is a realloc of the last alloc? */ + + if (!block || block->size - block->used < size) { + /* Slow path: have to allocate a new block. */ + block = upb_arena_allocblock(a, size); + + if (!block) { + return NULL; /* Out of memory. */ + } + } + + ret = (char*)block + block->used; + block->used += size; + + if (oldsize > 0) { + memcpy(ret, ptr, oldsize); /* Preserve existing data. */ + } + + /* TODO(haberman): ASAN unpoison. */ + + a->bytes_allocated += size; + return ret; +} + +/* Public Arena API ***********************************************************/ + +#define upb_alignof(type) offsetof (struct { char c; type member; }, member) + +upb_arena *upb_arena_init(void *mem, size_t n, upb_alloc *alloc) { + const size_t first_block_overhead = sizeof(upb_arena) + sizeof(mem_block); + upb_arena *a; + bool owned = false; + + /* Round block size down to alignof(*a) since we will allocate the arena + * itself at the end. */ + n &= ~(upb_alignof(upb_arena) - 1); + + if (n < first_block_overhead) { + /* We need to malloc the initial block. */ + n = first_block_overhead + 256; + owned = true; + if (!alloc || !(mem = upb_malloc(alloc, n))) { + return NULL; + } + } + + a = (void*)((char*)mem + n - sizeof(*a)); + n -= sizeof(*a); + + a->alloc.func = &upb_arena_doalloc; + a->block_alloc = &upb_alloc_global; + a->bytes_allocated = 0; + a->next_block_size = 256; + a->max_block_size = 16384; + a->cleanup_head = NULL; + a->block_head = NULL; + a->block_alloc = alloc; + + upb_arena_addblock(a, mem, n, owned); + + return a; +} + +#undef upb_alignof + +void upb_arena_free(upb_arena *a) { + cleanup_ent *ent = a->cleanup_head; + mem_block *block = a->block_head; + + while (ent) { + ent->cleanup(ent->ud); + ent = ent->next; + } + + /* Must do this after running cleanup functions, because this will delete + * the memory we store our cleanup entries in! */ + while (block) { + /* Load first since we are deleting block. */ + mem_block *next = block->next; + + if (block->owned) { + upb_free(a->block_alloc, block); + } + + block = next; + } +} + +bool upb_arena_addcleanup(upb_arena *a, void *ud, upb_cleanup_func *func) { + cleanup_ent *ent = upb_malloc(&a->alloc, sizeof(cleanup_ent)); + if (!ent) { + return false; /* Out of memory. */ + } + + ent->cleanup = func; + ent->ud = ud; + ent->next = a->cleanup_head; + a->cleanup_head = ent; + + return true; +} + +size_t upb_arena_bytesallocated(const upb_arena *a) { + return a->bytes_allocated; +} diff --git a/upb/upb.h b/upb/upb.h new file mode 100644 index 00000000000..79c19d281e9 --- /dev/null +++ b/upb/upb.h @@ -0,0 +1,364 @@ +/* +** This file contains shared definitions that are widely used across upb. +** +** This is a mixed C/C++ interface that offers a full API to both languages. +** See the top-level README for more information. +*/ + +#ifndef UPB_H_ +#define UPB_H_ + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +#include +namespace upb { +class Arena; +class Status; +template class InlinedArena; +} +#endif + +#include "upb/port_def.inc" + +/* upb_status *****************************************************************/ + +/* upb_status represents a success or failure status and error message. + * It owns no resources and allocates no memory, so it should work + * even in OOM situations. */ + +/* The maximum length of an error message before it will get truncated. */ +#define UPB_STATUS_MAX_MESSAGE 127 + +typedef struct { + bool ok; + char msg[UPB_STATUS_MAX_MESSAGE]; /* Error message; NULL-terminated. */ +} upb_status; + +#ifdef __cplusplus +extern "C" { +#endif + +const char *upb_status_errmsg(const upb_status *status); +bool upb_ok(const upb_status *status); + +/* Any of the functions that write to a status object allow status to be NULL, + * to support use cases where the function's caller does not care about the + * status message. */ +void upb_status_clear(upb_status *status); +void upb_status_seterrmsg(upb_status *status, const char *msg); +void upb_status_seterrf(upb_status *status, const char *fmt, ...); +void upb_status_vseterrf(upb_status *status, const char *fmt, va_list args); + +UPB_INLINE void upb_status_setoom(upb_status *status) { + upb_status_seterrmsg(status, "out of memory"); +} + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::Status { + public: + Status() { upb_status_clear(&status_); } + + upb_status* ptr() { return &status_; } + + /* Returns true if there is no error. */ + bool ok() const { return upb_ok(&status_); } + + /* Guaranteed to be NULL-terminated. */ + const char *error_message() const { return upb_status_errmsg(&status_); } + + /* The error message will be truncated if it is longer than + * UPB_STATUS_MAX_MESSAGE-4. */ + void SetErrorMessage(const char *msg) { upb_status_seterrmsg(&status_, msg); } + void SetFormattedErrorMessage(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + upb_status_vseterrf(&status_, fmt, args); + va_end(args); + } + + /* Resets the status to a successful state with no message. */ + void Clear() { upb_status_clear(&status_); } + + private: + upb_status status_; +}; + +#endif /* __cplusplus */ + +/** upb_strview ************************************************************/ + +typedef struct { + const char *data; + size_t size; +} upb_strview; + +UPB_INLINE upb_strview upb_strview_make(const char *data, size_t size) { + upb_strview ret; + ret.data = data; + ret.size = size; + return ret; +} + +UPB_INLINE upb_strview upb_strview_makez(const char *data) { + return upb_strview_make(data, strlen(data)); +} + +UPB_INLINE bool upb_strview_eql(upb_strview a, upb_strview b) { + return a.size == b.size && memcmp(a.data, b.data, a.size) == 0; +} + +#define UPB_STRVIEW_INIT(ptr, len) {ptr, len} + +#define UPB_STRVIEW_FORMAT "%.*s" +#define UPB_STRVIEW_ARGS(view) (int)(view).size, (view).data + +/** upb_alloc *****************************************************************/ + +/* A upb_alloc is a possibly-stateful allocator object. + * + * It could either be an arena allocator (which doesn't require individual + * free() calls) or a regular malloc() (which does). The client must therefore + * free memory unless it knows that the allocator is an arena allocator. */ + +struct upb_alloc; +typedef struct upb_alloc upb_alloc; + +/* A malloc()/free() function. + * If "size" is 0 then the function acts like free(), otherwise it acts like + * realloc(). Only "oldsize" bytes from a previous allocation are preserved. */ +typedef void *upb_alloc_func(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size); + +struct upb_alloc { + upb_alloc_func *func; +}; + +UPB_INLINE void *upb_malloc(upb_alloc *alloc, size_t size) { + UPB_ASSERT(alloc); + return alloc->func(alloc, NULL, 0, size); +} + +UPB_INLINE void *upb_realloc(upb_alloc *alloc, void *ptr, size_t oldsize, + size_t size) { + UPB_ASSERT(alloc); + return alloc->func(alloc, ptr, oldsize, size); +} + +UPB_INLINE void upb_free(upb_alloc *alloc, void *ptr) { + assert(alloc); + alloc->func(alloc, ptr, 0, 0); +} + +/* The global allocator used by upb. Uses the standard malloc()/free(). */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern upb_alloc upb_alloc_global; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +/* Functions that hard-code the global malloc. + * + * We still get benefit because we can put custom logic into our global + * allocator, like injecting out-of-memory faults in debug/testing builds. */ + +UPB_INLINE void *upb_gmalloc(size_t size) { + return upb_malloc(&upb_alloc_global, size); +} + +UPB_INLINE void *upb_grealloc(void *ptr, size_t oldsize, size_t size) { + return upb_realloc(&upb_alloc_global, ptr, oldsize, size); +} + +UPB_INLINE void upb_gfree(void *ptr) { + upb_free(&upb_alloc_global, ptr); +} + +/* upb_arena ******************************************************************/ + +/* upb_arena is a specific allocator implementation that uses arena allocation. + * The user provides an allocator that will be used to allocate the underlying + * arena blocks. Arenas by nature do not require the individual allocations + * to be freed. However the Arena does allow users to register cleanup + * functions that will run when the arena is destroyed. + * + * A upb_arena is *not* thread-safe. + * + * You could write a thread-safe arena allocator that satisfies the + * upb_alloc interface, but it would not be as efficient for the + * single-threaded case. */ + +typedef void upb_cleanup_func(void *ud); + +struct upb_arena; +typedef struct upb_arena upb_arena; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Creates an arena from the given initial block (if any -- n may be 0). + * Additional blocks will be allocated from |alloc|. If |alloc| is NULL, this + * is a fixed-size arena and cannot grow. */ +upb_arena *upb_arena_init(void *mem, size_t n, upb_alloc *alloc); +void upb_arena_free(upb_arena *a); +bool upb_arena_addcleanup(upb_arena *a, void *ud, upb_cleanup_func *func); +size_t upb_arena_bytesallocated(const upb_arena *a); + +UPB_INLINE upb_alloc *upb_arena_alloc(upb_arena *a) { return (upb_alloc*)a; } + +/* Convenience wrappers around upb_alloc functions. */ + +UPB_INLINE void *upb_arena_malloc(upb_arena *a, size_t size) { + return upb_malloc(upb_arena_alloc(a), size); +} + +UPB_INLINE void *upb_arena_realloc(upb_arena *a, void *ptr, size_t oldsize, + size_t size) { + return upb_realloc(upb_arena_alloc(a), ptr, oldsize, size); +} + +UPB_INLINE upb_arena *upb_arena_new(void) { + return upb_arena_init(NULL, 0, &upb_alloc_global); +} + +#ifdef __cplusplus +} /* extern "C" */ + +class upb::Arena { + public: + /* A simple arena with no initial memory block and the default allocator. */ + Arena() : ptr_(upb_arena_new(), upb_arena_free) {} + + upb_arena* ptr() { return ptr_.get(); } + + /* Allows this arena to be used as a generic allocator. + * + * The arena does not need free() calls so when using Arena as an allocator + * it is safe to skip them. However they are no-ops so there is no harm in + * calling free() either. */ + upb_alloc *allocator() { return upb_arena_alloc(ptr_.get()); } + + /* Add a cleanup function to run when the arena is destroyed. + * Returns false on out-of-memory. */ + bool AddCleanup(void *ud, upb_cleanup_func* func) { + return upb_arena_addcleanup(ptr_.get(), ud, func); + } + + /* Total number of bytes that have been allocated. It is undefined what + * Realloc() does to &arena_ counter. */ + size_t BytesAllocated() const { return upb_arena_bytesallocated(ptr_.get()); } + + private: + std::unique_ptr ptr_; +}; + +#endif + +/* upb::InlinedArena **********************************************************/ + +/* upb::InlinedArena seeds the arenas with a predefined amount of memory. No + * heap memory will be allocated until the initial block is exceeded. + * + * These types only exist in C++ */ + +#ifdef __cplusplus + +template class upb::InlinedArena : public upb::Arena { + public: + InlinedArena() : ptr_(upb_arena_new(&initial_block_, N, &upb_alloc_global)) {} + + upb_arena* ptr() { return ptr_.get(); } + + private: + InlinedArena(const InlinedArena*) = delete; + InlinedArena& operator=(const InlinedArena*) = delete; + + std::unique_ptr ptr_; + char initial_block_[N]; +}; + +#endif /* __cplusplus */ + +/* Constants ******************************************************************/ + +/* Generic function type. */ +typedef void upb_func(void); + +/* A list of types as they are encoded on-the-wire. */ +typedef enum { + UPB_WIRE_TYPE_VARINT = 0, + UPB_WIRE_TYPE_64BIT = 1, + UPB_WIRE_TYPE_DELIMITED = 2, + UPB_WIRE_TYPE_START_GROUP = 3, + UPB_WIRE_TYPE_END_GROUP = 4, + UPB_WIRE_TYPE_32BIT = 5 +} upb_wiretype_t; + +/* The types a field can have. Note that this list is not identical to the + * types defined in descriptor.proto, which gives INT32 and SINT32 separate + * types (we distinguish the two with the "integer encoding" enum below). */ +typedef enum { + /* Types stored in 1 byte. */ + UPB_TYPE_BOOL = 1, + /* Types stored in 4 bytes. */ + UPB_TYPE_FLOAT = 2, + UPB_TYPE_INT32 = 3, + UPB_TYPE_UINT32 = 4, + UPB_TYPE_ENUM = 5, /* Enum values are int32. */ + /* Types stored as pointers (probably 4 or 8 bytes). */ + UPB_TYPE_STRING = 6, + UPB_TYPE_BYTES = 7, + UPB_TYPE_MESSAGE = 8, + /* Types stored as 8 bytes. */ + UPB_TYPE_DOUBLE = 9, + UPB_TYPE_INT64 = 10, + UPB_TYPE_UINT64 = 11 +} upb_fieldtype_t; + +/* The repeated-ness of each field; this matches descriptor.proto. */ +typedef enum { + UPB_LABEL_OPTIONAL = 1, + UPB_LABEL_REQUIRED = 2, + UPB_LABEL_REPEATED = 3 +} upb_label_t; + +/* Descriptor types, as defined in descriptor.proto. */ +typedef enum { + UPB_DESCRIPTOR_TYPE_DOUBLE = 1, + UPB_DESCRIPTOR_TYPE_FLOAT = 2, + UPB_DESCRIPTOR_TYPE_INT64 = 3, + UPB_DESCRIPTOR_TYPE_UINT64 = 4, + UPB_DESCRIPTOR_TYPE_INT32 = 5, + UPB_DESCRIPTOR_TYPE_FIXED64 = 6, + UPB_DESCRIPTOR_TYPE_FIXED32 = 7, + UPB_DESCRIPTOR_TYPE_BOOL = 8, + UPB_DESCRIPTOR_TYPE_STRING = 9, + UPB_DESCRIPTOR_TYPE_GROUP = 10, + UPB_DESCRIPTOR_TYPE_MESSAGE = 11, + UPB_DESCRIPTOR_TYPE_BYTES = 12, + UPB_DESCRIPTOR_TYPE_UINT32 = 13, + UPB_DESCRIPTOR_TYPE_ENUM = 14, + UPB_DESCRIPTOR_TYPE_SFIXED32 = 15, + UPB_DESCRIPTOR_TYPE_SFIXED64 = 16, + UPB_DESCRIPTOR_TYPE_SINT32 = 17, + UPB_DESCRIPTOR_TYPE_SINT64 = 18 +} upb_descriptortype_t; + +extern const uint8_t upb_desctype_to_fieldtype[]; + +#include "upb/port_undef.inc" + +#endif /* UPB_H_ */ diff --git a/upbc/generator.cc b/upbc/generator.cc new file mode 100644 index 00000000000..7096278180b --- /dev/null +++ b/upbc/generator.cc @@ -0,0 +1,826 @@ + +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "google/protobuf/io/zero_copy_stream.h" + +#include "upbc/generator.h" +#include "upbc/message_layout.h" + +namespace protoc = ::google::protobuf::compiler; +namespace protobuf = ::google::protobuf; + +static std::string StripExtension(absl::string_view fname) { + size_t lastdot = fname.find_last_of("."); + if (lastdot == std::string::npos) { + return std::string(fname); + } + return std::string(fname.substr(0, lastdot)); +} + +static std::string HeaderFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upb.h"; +} + +static std::string SourceFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upb.c"; +} + +static std::string DefHeaderFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upbdefs.h"; +} + +static std::string DefSourceFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upbdefs.c"; +} + +class Output { + public: + Output(protobuf::io::ZeroCopyOutputStream* stream) : stream_(stream) {} + ~Output() { stream_->BackUp(size_); } + + template + void operator()(absl::string_view format, const Arg&... arg) { + Write(absl::Substitute(format, arg...)); + } + + private: + void Write(absl::string_view data) { + while (!data.empty()) { + RefreshOutput(); + size_t to_write = std::min(data.size(), size_); + memcpy(ptr_, data.data(), to_write); + data.remove_prefix(to_write); + ptr_ += to_write; + size_ -= to_write; + } + } + + void RefreshOutput() { + while (size_ == 0) { + void *ptr; + int size; + if (!stream_->Next(&ptr, &size)) { + fprintf(stderr, "upbc: Failed to write to to output\n"); + abort(); + } + ptr_ = static_cast(ptr); + size_ = size; + } + } + + protobuf::io::ZeroCopyOutputStream* stream_; + char *ptr_ = nullptr; + size_t size_ = 0; +}; + +namespace upbc { + +class Generator : public protoc::CodeGenerator { + ~Generator() override {} + bool Generate(const protobuf::FileDescriptor* file, + const std::string& parameter, protoc::GeneratorContext* context, + std::string* error) const override; + +}; + +void AddMessages(const protobuf::Descriptor* message, + std::vector* messages) { + messages->push_back(message); + for (int i = 0; i < message->nested_type_count(); i++) { + AddMessages(message->nested_type(i), messages); + } +} + +void AddEnums(const protobuf::Descriptor* message, + std::vector* enums) { + for (int i = 0; i < message->enum_type_count(); i++) { + enums->push_back(message->enum_type(i)); + } + for (int i = 0; i < message->nested_type_count(); i++) { + AddEnums(message->nested_type(i), enums); + } +} + +template +void SortDefs(std::vector* defs) { + std::sort(defs->begin(), defs->end(), + [](T a, T b) { return a->full_name() < b->full_name(); }); +} + +std::vector SortedMessages( + const protobuf::FileDescriptor* file) { + std::vector messages; + for (int i = 0; i < file->message_type_count(); i++) { + AddMessages(file->message_type(i), &messages); + } + return messages; +} + +std::vector SortedEnums( + const protobuf::FileDescriptor* file) { + std::vector enums; + for (int i = 0; i < file->enum_type_count(); i++) { + enums.push_back(file->enum_type(i)); + } + for (int i = 0; i < file->message_type_count(); i++) { + AddEnums(file->message_type(i), &enums); + } + SortDefs(&enums); + return enums; +} + +std::vector FieldNumberOrder( + const protobuf::Descriptor* message) { + std::vector messages; + for (int i = 0; i < message->field_count(); i++) { + messages.push_back(message->field(i)); + } + std::sort(messages.begin(), messages.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return a->number() < b->number(); + }); + return messages; +} + +std::vector SortedSubmessages( + const protobuf::Descriptor* message) { + std::vector ret; + for (int i = 0; i < message->field_count(); i++) { + if (message->field(i)->cpp_type() == + protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + ret.push_back(message->field(i)); + } + } + std::sort(ret.begin(), ret.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return a->message_type()->full_name() < + b->message_type()->full_name(); + }); + return ret; +} + +std::string ToCIdent(absl::string_view str) { + return absl::StrReplaceAll(str, {{".", "_"}, {"/", "_"}}); +} + +std::string DefInitSymbol(const protobuf::FileDescriptor *file) { + return ToCIdent(file->name()) + "_upbdefinit"; +} + +std::string ToPreproc(absl::string_view str) { + return absl::AsciiStrToUpper(ToCIdent(str)); +} + +std::string EnumValueSymbol(const protobuf::EnumValueDescriptor* value) { + return ToCIdent(value->full_name()); +} + +std::string GetSizeInit(const MessageLayout::Size& size) { + return absl::Substitute("UPB_SIZE($0, $1)", size.size32, size.size64); +} + +std::string MessageName(const protobuf::Descriptor* descriptor) { + return ToCIdent(descriptor->full_name()); +} + +std::string MessageInit(const protobuf::Descriptor* descriptor) { + return MessageName(descriptor) + "_msginit"; +} + +std::string CTypeInternal(const protobuf::FieldDescriptor* field, + bool is_const) { + std::string maybe_const = is_const ? "const " : ""; + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + std::string maybe_struct = + field->file() != field->message_type()->file() ? "struct " : ""; + return maybe_const + maybe_struct + MessageName(field->message_type()) + + "*"; + } + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return "bool"; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return "float"; + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return "int32_t"; + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return "uint32_t"; + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return "double"; + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return "int64_t"; + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return "uint64_t"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return "upb_strview"; + default: + fprintf(stderr, "Unexpected type"); + abort(); + } +} + +std::string UpbType(const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return "UPB_TYPE_MESSAGE"; + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return "UPB_TYPE_ENUM"; + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return "UPB_TYPE_BOOL"; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return "UPB_TYPE_FLOAT"; + case protobuf::FieldDescriptor::CPPTYPE_INT32: + return "UPB_TYPE_INT32"; + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return "UPB_TYPE_UINT32"; + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return "UPB_TYPE_DOUBLE"; + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return "UPB_TYPE_INT64"; + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return "UPB_TYPE_UINT64"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return "UPB_TYPE_STRING"; + default: + fprintf(stderr, "Unexpected type"); + abort(); + } +} + +std::string FieldDefault(const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return "NULL"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return absl::Substitute("upb_strview_make(\"$0\", strlen(\"$0\"))", + absl::CEscape(field->default_value_string())); + case protobuf::FieldDescriptor::CPPTYPE_INT32: + return absl::StrCat(field->default_value_int32()); + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return absl::StrCat(field->default_value_int64()); + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return absl::StrCat(field->default_value_uint32()); + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return absl::StrCat(field->default_value_uint64()); + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return absl::StrCat(field->default_value_float()); + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return absl::StrCat(field->default_value_double()); + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return field->default_value_bool() ? "true" : "false"; + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return EnumValueSymbol(field->default_value_enum()); + } + ABSL_ASSERT(false); + return "XXX"; +} + +std::string CType(const protobuf::FieldDescriptor* field) { + return CTypeInternal(field, false); +} + +std::string CTypeConst(const protobuf::FieldDescriptor* field) { + return CTypeInternal(field, true); +} + +void DumpEnumValues(const protobuf::EnumDescriptor* desc, Output& output) { + std::vector values; + for (int i = 0; i < desc->value_count(); i++) { + values.push_back(desc->value(i)); + } + std::sort(values.begin(), values.end(), + [](const protobuf::EnumValueDescriptor* a, + const protobuf::EnumValueDescriptor* b) { + return a->number() < b->number(); + }); + + for (size_t i = 0; i < values.size(); i++) { + auto value = values[i]; + output(" $0 = $1", EnumValueSymbol(value), value->number()); + if (i != values.size() - 1) { + output(","); + } + output("\n"); + } +} + +void EmitFileWarning(const protobuf::FileDescriptor* file, Output& output) { + output( + "/* This file was generated by upbc (the upb compiler) from the input\n" + " * file:\n" + " *\n" + " * $0\n" + " *\n" + " * Do not edit -- your changes will be discarded when the file is\n" + " * regenerated. */\n\n", + file->name()); +} + +void GenerateMessageInHeader(const protobuf::Descriptor* message, Output& output) { + MessageLayout layout(message); + + output("/* $0 */\n\n", message->full_name()); + std::string msgname = ToCIdent(message->full_name()); + output( + "UPB_INLINE $0 *$0_new(upb_arena *arena) {\n" + " return ($0 *)upb_msg_new(&$1, arena);\n" + "}\n" + "UPB_INLINE $0 *$0_parse(const char *buf, size_t size,\n" + " upb_arena *arena) {\n" + " $0 *ret = $0_new(arena);\n" + " return (ret && upb_decode(buf, size, ret, &$1, arena)) ? ret : NULL;\n" + "}\n" + "UPB_INLINE char *$0_serialize(const $0 *msg, upb_arena *arena, size_t " + "*len) {\n" + " return upb_encode(msg, &$1, arena, len);\n" + "}\n" + "\n", + MessageName(message), MessageInit(message)); + + for (int i = 0; i < message->oneof_decl_count(); i++) { + const protobuf::OneofDescriptor* oneof = message->oneof_decl(i); + std::string fullname = ToCIdent(oneof->full_name()); + output("typedef enum {\n"); + for (int j = 0; j < oneof->field_count(); j++) { + const protobuf::FieldDescriptor* field = oneof->field(j); + output(" $0_$1 = $2,\n", fullname, field->name(), field->number()); + } + output( + " $0_NOT_SET = 0\n" + "} $0_oneofcases;\n", + fullname); + output( + "UPB_INLINE $0_oneofcases $1_$2_case(const $1* msg) { " + "return ($0_oneofcases)UPB_FIELD_AT(msg, int32_t, $3); }\n" + "\n", + fullname, msgname, oneof->name(), + GetSizeInit(layout.GetOneofCaseOffset(oneof))); + } + + for (auto field : FieldNumberOrder(message)) { + + if (layout.HasHasbit(field)) { + output( + "UPB_INLINE bool $0_has_$1(const $0 *msg) { " + "return _upb_has_field(msg, $2); }\n", + msgname, field->name(), layout.GetHasbitIndex(field)); + } else if (field->containing_oneof()) { + output( + "UPB_INLINE bool $0_has_$1(const $0 *msg) { " + "return _upb_has_oneof_field(msg, $2, $3); }\n", + msgname, field->name(), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number()); + } + + if (field->is_repeated()) { + output( + "UPB_INLINE $0 const* $1_$2(const $1 *msg, size_t *len) { " + "return ($0 const*)_upb_array_accessor(msg, $3, len); }\n", + CTypeConst(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field))); + } else if (field->containing_oneof()) { + output( + "UPB_INLINE $0 $1_$2(const $1 *msg) { " + "return UPB_READ_ONEOF(msg, $0, $3, $4, $5, $6); }\n", + CTypeConst(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number(), FieldDefault(field)); + } else { + output( + "UPB_INLINE $0 $1_$2(const $1 *msg) { " + "return UPB_FIELD_AT(msg, $0, $3); }\n", + CTypeConst(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field))); + } + } + + output("\n"); + + for (auto field : FieldNumberOrder(message)) { + if (field->is_repeated()) { + output( + "UPB_INLINE $0* $1_mutable_$2($1 *msg, size_t *len) {\n" + " return ($0*)_upb_array_mutable_accessor(msg, $3, len);\n" + "}\n", + CType(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field))); + output( + "UPB_INLINE $0* $1_resize_$2($1 *msg, size_t len, " + "upb_arena *arena) {\n" + " return ($0*)_upb_array_resize_accessor(msg, $3, len, $4, $5, " + "arena);\n" + "}\n", + CType(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(MessageLayout::SizeOfUnwrapped(field).size), + UpbType(field)); + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + output( + "UPB_INLINE struct $0* $1_add_$2($1 *msg, upb_arena *arena) {\n" + " struct $0* sub = (struct $0*)upb_msg_new(&$3, arena);\n" + " bool ok = _upb_array_append_accessor(\n" + " msg, $4, $5, $6, &sub, arena);\n" + " if (!ok) return NULL;\n" + " return sub;\n" + "}\n", + MessageName(field->message_type()), msgname, field->name(), + MessageInit(field->message_type()), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(MessageLayout::SizeOfUnwrapped(field).size), + UpbType(field)); + } else { + output( + "UPB_INLINE bool $1_add_$2($1 *msg, $0 val, upb_arena *arena) {\n" + " return _upb_array_append_accessor(\n" + " msg, $3, $4, $5, &val, arena);\n" + "}\n", + CType(field), msgname, field->name(), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(MessageLayout::SizeOfUnwrapped(field).size), + UpbType(field)); + } + } else { + output("UPB_INLINE void $0_set_$1($0 *msg, $2 value) {\n", msgname, + field->name(), CType(field)); + if (field->containing_oneof()) { + output( + " UPB_WRITE_ONEOF(msg, $0, $1, value, $2, $3);\n" + "}\n", + CType(field), GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number()); + } else { + if (MessageLayout::HasHasbit(field)) { + output(" _upb_sethas(msg, $0);\n", layout.GetHasbitIndex(field)); + } + output( + " UPB_FIELD_AT(msg, $0, $1) = value;\n" + "}\n", + CType(field), GetSizeInit(layout.GetFieldOffset(field))); + } + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + output( + "UPB_INLINE struct $0* $1_mutable_$2($1 *msg, upb_arena *arena) {\n" + " struct $0* sub = (struct $0*)$1_$2(msg);\n" + " if (sub == NULL) {\n" + " sub = (struct $0*)upb_msg_new(&$3, arena);\n" + " if (!sub) return NULL;\n" + " $1_set_$2(msg, sub);\n" + " }\n" + " return sub;\n" + "}\n", + MessageName(field->message_type()), msgname, field->name(), + MessageInit(field->message_type())); + } + } + } + + output("\n"); +} + +void WriteHeader(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + output( + "#ifndef $0_UPB_H_\n" + "#define $0_UPB_H_\n\n" + "#include \"upb/generated_util.h\"\n" + "#include \"upb/msg.h\"\n" + "#include \"upb/decode.h\"\n" + "#include \"upb/encode.h\"\n\n", + ToPreproc(file->name())); + + for (int i = 0; i < file->public_dependency_count(); i++) { + const auto& name = file->public_dependency(i)->name(); + if (i == 0) { + output("/* Public Imports. */\n"); + } + output("#include \"$0\"\n", HeaderFilename(name)); + if (i == file->public_dependency_count() - 1) { + output("\n"); + } + } + + output( + "#include \"upb/port_def.inc\"\n" + "\n" + "#ifdef __cplusplus\n" + "extern \"C\" {\n" + "#endif\n" + "\n"); + + std::vector this_file_messages = + SortedMessages(file); + + // Forward-declare types defined in this file. + for (auto message : this_file_messages) { + output("struct $0;\n", ToCIdent(message->full_name())); + } + for (auto message : this_file_messages) { + output("typedef struct $0 $0;\n", ToCIdent(message->full_name())); + } + for (auto message : this_file_messages) { + output("extern const upb_msglayout $0;\n", MessageInit(message)); + } + + // Forward-declare types not in this file, but used as submessages. + // Order by full name for consistent ordering. + std::map forward_messages; + + for (auto message : SortedMessages(file)) { + for (int i = 0; i < message->field_count(); i++) { + const protobuf::FieldDescriptor* field = message->field(i); + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->file() != field->message_type()->file()) { + forward_messages[field->message_type()->full_name()] = + field->message_type(); + } + } + } + for (const auto& pair : forward_messages) { + output("struct $0;\n", MessageName(pair.second)); + } + for (const auto& pair : forward_messages) { + output("extern const upb_msglayout $0;\n", MessageInit(pair.second)); + } + + if (!this_file_messages.empty()) { + output("\n"); + } + + std::vector this_file_enums = + SortedEnums(file); + + for (auto enumdesc : this_file_enums) { + output("typedef enum {\n"); + DumpEnumValues(enumdesc, output); + output("} $0;\n\n", ToCIdent(enumdesc->full_name())); + } + + output("\n"); + + for (auto message : this_file_messages) { + GenerateMessageInHeader(message, output); + } + + output( + "#ifdef __cplusplus\n" + "} /* extern \"C\" */\n" + "#endif\n" + "\n" + "#include \"upb/port_undef.inc\"\n" + "\n" + "#endif /* $0_UPB_H_ */\n", + ToPreproc(file->name())); +} + +void WriteSource(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output( + "#include \n" + "#include \"upb/msg.h\"\n" + "#include \"$0\"\n", + HeaderFilename(file->name())); + + for (int i = 0; i < file->dependency_count(); i++) { + output("#include \"$0\"\n", HeaderFilename(file->dependency(i)->name())); + } + + output( + "\n" + "#include \"upb/port_def.inc\"\n" + "\n"); + + + for (auto message : SortedMessages(file)) { + std::string msgname = ToCIdent(message->full_name()); + std::string fields_array_ref = "NULL"; + std::string submsgs_array_ref = "NULL"; + std::string oneofs_array_ref = "NULL"; + absl::flat_hash_map submsg_indexes; + MessageLayout layout(message); + std::vector sorted_submsgs = + SortedSubmessages(message); + + if (!sorted_submsgs.empty()) { + // TODO(haberman): could save a little bit of space by only generating a + // "submsgs" array for every strongly-connected component. + std::string submsgs_array_name = msgname + "_submsgs"; + submsgs_array_ref = "&" + submsgs_array_name + "[0]"; + output("static const upb_msglayout *const $0[$1] = {\n", + submsgs_array_name, sorted_submsgs.size()); + + int i = 0; + for (auto submsg : sorted_submsgs) { + if (submsg_indexes.find(submsg->message_type()) != + submsg_indexes.end()) { + continue; + } + output(" &$0,\n", MessageInit(submsg->message_type())); + submsg_indexes[submsg->message_type()] = i++; + } + + output("};\n\n"); + } + + std::vector field_number_order = + FieldNumberOrder(message); + if (!field_number_order.empty()) { + std::string fields_array_name = msgname + "__fields"; + fields_array_ref = "&" + fields_array_name + "[0]"; + output("static const upb_msglayout_field $0[$1] = {\n", + fields_array_name, field_number_order.size()); + for (auto field : field_number_order) { + int submsg_index = 0; + std::string presence = "0"; + + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + submsg_index = submsg_indexes[field->message_type()]; + } + + if (MessageLayout::HasHasbit(field)) { + presence = absl::StrCat(layout.GetHasbitIndex(field)); + } else if (field->containing_oneof()) { + MessageLayout::Size case_offset = + layout.GetOneofCaseOffset(field->containing_oneof()); + + // Our encoding that distinguishes oneofs from presence-having fields. + case_offset.size32 = -case_offset.size32 - 1; + case_offset.size64 = -case_offset.size64 - 1; + presence = GetSizeInit(case_offset); + } + + output(" {$0, $1, $2, $3, $4, $5},\n", + field->number(), + GetSizeInit(layout.GetFieldOffset(field)), + presence, + submsg_index, + field->type(), + field->label()); + } + output("};\n\n"); + } + + output("const upb_msglayout $0 = {\n", MessageInit(message)); + output(" $0,\n", submsgs_array_ref); + output(" $0,\n", fields_array_ref); + output(" $0, $1, $2,\n", GetSizeInit(layout.message_size()), + field_number_order.size(), + "false" // TODO: extendable + ); + + output("};\n\n"); + } + + output("#include \"upb/port_undef.inc\"\n"); + output("\n"); +} + +void GenerateMessageDefAccessor(const protobuf::Descriptor* d, Output& output) { + output("UPB_INLINE const upb_msgdef *$0_getmsgdef(upb_symtab *s) {\n", + ToCIdent(d->full_name())); + output(" _upb_symtab_loaddefinit(s, &$0);\n", DefInitSymbol(d->file())); + output(" return upb_symtab_lookupmsg(s, \"$0\");\n", d->full_name()); + output("}\n"); + output("\n"); + + for (int i = 0; i < d->nested_type_count(); i++) { + GenerateMessageDefAccessor(d->nested_type(i), output); + } +} + +void WriteDefHeader(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output( + "#ifndef $0_UPBDEFS_H_\n" + "#define $0_UPBDEFS_H_\n\n" + "#include \"upb/def.h\"\n" + "#include \"upb/port_def.inc\"\n" + "#ifdef __cplusplus\n" + "extern \"C\" {\n" + "#endif\n\n", + ToPreproc(file->name())); + + output("#include \"upb/def.h\"\n"); + output("\n"); + output("#include \"upb/port_def.inc\"\n"); + output("\n"); + + output("extern upb_def_init $0;\n", DefInitSymbol(file)); + output("\n"); + + for (int i = 0; i < file->message_type_count(); i++) { + GenerateMessageDefAccessor(file->message_type(i), output); + } + + output( + "#ifdef __cplusplus\n" + "} /* extern \"C\" */\n" + "#endif\n" + "\n" + "#include \"upb/port_undef.inc\"\n" + "\n" + "#endif /* $0_UPBDEFS_H_ */\n", + ToPreproc(file->name())); +} + +// Escape C++ trigraphs by escaping question marks to \? +std::string EscapeTrigraphs(absl::string_view to_escape) { + return absl::StrReplaceAll(to_escape, {{"?", "\\?"}}); +} + +void WriteDefSource(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output("#include \"upb/def.h\"\n"); + output("\n"); + + for (int i = 0; i < file->dependency_count(); i++) { + output("extern upb_def_init $0;\n", DefInitSymbol(file->dependency(i))); + } + + protobuf::FileDescriptorProto file_proto; + file->CopyTo(&file_proto); + std::string file_data; + file_proto.SerializeToString(&file_data); + + output("static const char descriptor[$0] =\n", file_data.size()); + + { + if (file_data.size() > 65535) { + // Workaround for MSVC: "Error C1091: compiler limit: string exceeds + // 65535 bytes in length". Declare a static array of chars rather than + // use a string literal. Only write 25 bytes per line. + static const size_t kBytesPerLine = 25; + output("{ "); + for (size_t i = 0; i < file_data.size();) { + for (size_t j = 0; j < kBytesPerLine && i < file_data.size(); ++i, ++j) { + output("'$0', ", absl::CEscape(file_data.substr(i, 1))); + } + output("\n"); + } + output("'\\0' }"); // null-terminate + } else { + // Only write 40 bytes per line. + static const size_t kBytesPerLine = 40; + for (size_t i = 0; i < file_data.size(); i += kBytesPerLine) { + output( + "\"$0\"\n", + EscapeTrigraphs(absl::CEscape(file_data.substr(i, kBytesPerLine)))); + } + } + output(";\n"); + } + + output("static upb_def_init *deps[$0] = {\n", file->dependency_count() + 1); + for (int i = 0; i < file->dependency_count(); i++) { + output(" &$0,\n", DefInitSymbol(file->dependency(i))); + } + output(" NULL\n"); + output("};\n"); + + output("upb_def_init $0 = {\n", DefInitSymbol(file)); + output(" deps,\n"); + output(" \"$0\",\n", file->name()); + output(" UPB_STRVIEW_INIT(descriptor, $0)\n", file_data.size()); + output("};\n"); +} + +bool Generator::Generate(const protobuf::FileDescriptor* file, + const std::string& parameter, + protoc::GeneratorContext* context, + std::string* error) const { + Output h_output(context->Open(HeaderFilename(file->name()))); + WriteHeader(file, h_output); + + Output c_output(context->Open(SourceFilename(file->name()))); + WriteSource(file, c_output); + + Output h_def_output(context->Open(DefHeaderFilename(file->name()))); + WriteDefHeader(file, h_def_output); + + Output c_def_output(context->Open(DefSourceFilename(file->name()))); + WriteDefSource(file, c_def_output); + + return true; +} + +std::unique_ptr GetGenerator() { + return std::unique_ptr( + new Generator()); +} + +} // namespace upbc diff --git a/upbc/generator.h b/upbc/generator.h new file mode 100644 index 00000000000..ed6cedc6c77 --- /dev/null +++ b/upbc/generator.h @@ -0,0 +1,12 @@ + +#ifndef UPBC_GENERATOR_H_ +#define UPBC_GENERATOR_H_ + +#include +#include + +namespace upbc { +std::unique_ptr GetGenerator(); +} + +#endif // UPBC_GENERATOR_H_ diff --git a/upbc/main.cc b/upbc/main.cc new file mode 100644 index 00000000000..a9682a9c1a3 --- /dev/null +++ b/upbc/main.cc @@ -0,0 +1,9 @@ + +#include + +#include "upbc/generator.h" + +int main(int argc, char** argv) { + return google::protobuf::compiler::PluginMain(argc, argv, + upbc::GetGenerator().get()); +} diff --git a/upbc/message_layout.cc b/upbc/message_layout.cc new file mode 100644 index 00000000000..f0a68725c20 --- /dev/null +++ b/upbc/message_layout.cc @@ -0,0 +1,179 @@ + +#include "upbc/message_layout.h" + +namespace upbc { + +namespace protobuf = ::google::protobuf; + +static int64_t DivRoundUp(int64_t a, int64_t b) { + ABSL_ASSERT(a >= 0); + ABSL_ASSERT(b > 0); + return (a + b - 1) / b; +} + +MessageLayout::Size MessageLayout::Place( + MessageLayout::SizeAndAlign size_and_align) { + Size offset = size_; + offset.AlignUp(size_and_align.align); + size_ = offset; + size_.Add(size_and_align.size); + //maxalign_.MaxFrom(size_and_align.align); + maxalign_.MaxFrom(size_and_align.size); + return offset; +} + +bool MessageLayout::HasHasbit(const protobuf::FieldDescriptor* field) { + return field->file()->syntax() == protobuf::FileDescriptor::SYNTAX_PROTO2 && + field->label() != protobuf::FieldDescriptor::LABEL_REPEATED && + !field->containing_oneof(); +} + +MessageLayout::SizeAndAlign MessageLayout::SizeOf( + const protobuf::FieldDescriptor* field) { + if (field->is_repeated()) { + return {{4, 8}, {4, 8}}; // Pointer to array object. + } else { + return SizeOfUnwrapped(field); + } +} + +MessageLayout::SizeAndAlign MessageLayout::SizeOfUnwrapped( + const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return {{4, 8}, {4, 8}}; // Pointer to message. + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return {{8, 16}, {4, 8}}; // upb_stringview + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return {{1, 1}, {1, 1}}; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return {{4, 4}, {4, 4}}; + default: + return {{8, 8}, {8, 8}}; + } +} + +int64_t MessageLayout::FieldLayoutRank(const protobuf::FieldDescriptor* field) { + // Order: + // 1, 2, 3. primitive fields (8, 4, 1 byte) + // 4. string fields + // 5. submessage fields + // 6. repeated fields + // + // This has the following nice properties: + // + // 1. padding alignment is (nearly) minimized. + // 2. fields that might have defaults (1-4) are segregated + // from fields that are always zero-initialized (5-7). + // + // We skip oneof fields, because they are emitted in a separate pass. + int64_t rank; + if (field->containing_oneof()) { + fprintf(stderr, "shouldn't have oneofs here.\n"); + abort(); + } else if (field->label() == protobuf::FieldDescriptor::LABEL_REPEATED) { + rank = 6; + } else { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + rank = 5; + break; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + rank = 4; + break; + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + rank = 3; + break; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + rank = 2; + break; + default: + rank = 1; + break; + } + } + + // Break ties with field number. + return (rank << 29) | field->number(); +} + +void MessageLayout::ComputeLayout(const protobuf::Descriptor* descriptor) { + size_ = Size{0, 0}; + maxalign_ = Size{0, 0}; + PlaceNonOneofFields(descriptor); + PlaceOneofFields(descriptor); + + // Align overall size up to max size. + size_.AlignUp(maxalign_); +} + +void MessageLayout::PlaceNonOneofFields( + const protobuf::Descriptor* descriptor) { + std::vector field_order; + for (int i = 0; i < descriptor->field_count(); i++) { + const protobuf::FieldDescriptor* field = descriptor->field(i); + if (!field->containing_oneof()) { + field_order.push_back(descriptor->field(i)); + } + } + std::sort(field_order.begin(), field_order.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return FieldLayoutRank(a) < FieldLayoutRank(b); + }); + + // Place/count hasbits. + int hasbit_count = 0; + for (auto field : field_order) { + if (HasHasbit(field)) { + // We don't use hasbit 0, so that 0 can indicate "no presence" in the + // table. This wastes one hasbit, but we don't worry about it for now. + hasbit_indexes_[field] = ++hasbit_count; + } + } + + // Place hasbits at the beginning. + int64_t hasbit_bytes = DivRoundUp(hasbit_count, 8); + Place(SizeAndAlign{{hasbit_bytes, hasbit_bytes}, {1, 1}}); + + // Place non-oneof fields. + for (auto field : field_order) { + field_offsets_[field] = Place(SizeOf(field)); + } +} + +void MessageLayout::PlaceOneofFields(const protobuf::Descriptor* descriptor) { + std::vector oneof_order; + for (int i = 0; i < descriptor->oneof_decl_count(); i++) { + oneof_order.push_back(descriptor->oneof_decl(i)); + } + std::sort(oneof_order.begin(), oneof_order.end(), + [](const protobuf::OneofDescriptor* a, + const protobuf::OneofDescriptor* b) { + return a->full_name() < b->full_name(); + }); + + for (auto oneof : oneof_order) { + SizeAndAlign oneof_maxsize{{0, 0}, {0, 0}}; + // Calculate max size. + for (int i = 0; i < oneof->field_count(); i++) { + oneof_maxsize.MaxFrom(SizeOf(oneof->field(i))); + } + + // Place discriminator enum and data. + Size data = Place(oneof_maxsize); + Size discriminator = Place(SizeAndAlign{{4, 4}, {4, 4}}); + + oneof_case_offsets_[oneof] = discriminator; + + for (int i = 0; i < oneof->field_count(); i++) { + field_offsets_[oneof->field(i)] = data; + } + } +} + +} // namespace upbc diff --git a/upbc/message_layout.h b/upbc/message_layout.h new file mode 100644 index 00000000000..c2446a08656 --- /dev/null +++ b/upbc/message_layout.h @@ -0,0 +1,107 @@ + +#ifndef UPBC_MESSAGE_LAYOUT_H +#define UPBC_MESSAGE_LAYOUT_H + +#include "absl/base/macros.h" +#include "absl/container/flat_hash_map.h" +#include "google/protobuf/descriptor.h" + +namespace upbc { + +class MessageLayout { + public: + struct Size { + void Add(const Size& other) { + size32 += other.size32; + size64 += other.size64; + } + + void MaxFrom(const Size& other) { + size32 = std::max(size32, other.size32); + size64 = std::max(size64, other.size64); + } + + void AlignUp(const Size& align) { + size32 = Align(size32, align.size32); + size64 = Align(size64, align.size64); + } + + int64_t size32; + int64_t size64; + }; + + struct SizeAndAlign { + Size size; + Size align; + + void MaxFrom(const SizeAndAlign& other) { + size.MaxFrom(other.size); + align.MaxFrom(other.align); + } + }; + + MessageLayout(const google::protobuf::Descriptor* descriptor) { + ComputeLayout(descriptor); + } + + Size GetFieldOffset(const google::protobuf::FieldDescriptor* field) const { + return GetMapValue(field_offsets_, field); + } + + Size GetOneofCaseOffset( + const google::protobuf::OneofDescriptor* oneof) const { + return GetMapValue(oneof_case_offsets_, oneof); + } + + int GetHasbitIndex(const google::protobuf::FieldDescriptor* field) const { + return GetMapValue(hasbit_indexes_, field); + } + + Size message_size() const { return size_; } + + static bool HasHasbit(const google::protobuf::FieldDescriptor* field); + static SizeAndAlign SizeOfUnwrapped( + const google::protobuf::FieldDescriptor* field); + + private: + void ComputeLayout(const google::protobuf::Descriptor* descriptor); + void PlaceNonOneofFields(const google::protobuf::Descriptor* descriptor); + void PlaceOneofFields(const google::protobuf::Descriptor* descriptor); + Size Place(SizeAndAlign size_and_align); + + template + static V GetMapValue(const absl::flat_hash_map& map, K key) { + auto iter = map.find(key); + if (iter == map.end()) { + fprintf(stderr, "No value for field.\n"); + abort(); + } + return iter->second; + } + + static bool IsPowerOfTwo(size_t val) { + return (val & (val - 1)) == 0; + } + + static size_t Align(size_t val, size_t align) { + ABSL_ASSERT(IsPowerOfTwo(align)); + return (val + align - 1) & ~(align - 1); + } + + static SizeAndAlign SizeOf(const google::protobuf::FieldDescriptor* field); + static int64_t FieldLayoutRank( + const google::protobuf::FieldDescriptor* field); + + absl::flat_hash_map + field_offsets_; + absl::flat_hash_map + hasbit_indexes_; + absl::flat_hash_map + oneof_case_offsets_; + Size maxalign_; + Size size_; +}; + +} // namespace upbc + +#endif // UPBC_MESSAGE_LAYOUT_H From 3639d126693d1892fad1548844e7782102af23ca Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 21 Oct 2019 17:32:14 -0700 Subject: [PATCH 22/27] Bump version to 1.24.3 --- BUILD | 2 +- build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 182650c5e93..8fd76889fb0 100644 --- a/BUILD +++ b/BUILD @@ -81,7 +81,7 @@ g_stands_for = "ganges" core_version = "7.0.0" -version = "1.24.2" +version = "1.24.3" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index c1148806ea6..b86aaed6fb4 100644 --- a/build.yaml +++ b/build.yaml @@ -15,7 +15,7 @@ settings: core_version: 8.0.0 csharp_major_version: 2 g_stands_for: ganges - version: 1.24.2 + version: 1.24.3 filegroups: - name: alts_tsi headers: From be7bea389569890e932b1e011c0b5bfba85520b1 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 21 Oct 2019 17:32:47 -0700 Subject: [PATCH 23/27] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 4 ++-- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 35 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ecd7c460799..49de6e535ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.24.2") +set(PACKAGE_VERSION "1.24.3") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index ceb21b20a18..0e09549d11b 100644 --- a/Makefile +++ b/Makefile @@ -461,8 +461,8 @@ Q = @ endif CORE_VERSION = 8.0.0 -CPP_VERSION = 1.24.2 -CSHARP_VERSION = 2.24.2 +CPP_VERSION = 1.24.3 +CSHARP_VERSION = 2.24.3 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 5845fd9a3c6..55d525550af 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.24.2' + # version = '1.24.3' version = '0.0.9' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.24.2' + grpc_version = '1.24.3' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 78811e7ea16..bc85031a2d3 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.24.2' + version = '1.24.3' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index b5a5f691883..0d687266281 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.24.2' + version = '1.24.3' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 13dbd06e51d..97d1337bf63 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.24.2' + version = '1.24.3' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 3eb9d5009db..f2a0103220c 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.24.2' + version = '1.24.3' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 935ecb6951a..8d54e06c7e0 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.24.2 - 1.24.2 + 1.24.3 + 1.24.3 stable diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 6a692d7c756..fdaf2d37d79 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.24.2"; } +grpc::string Version() { return "1.24.3"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index 0e67c9d0567..5d8b06a3e7b 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "2.24.2.0"; + public const string CurrentAssemblyFileVersion = "2.24.3.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.24.2"; + public const string CurrentVersion = "2.24.3"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 905f7fefe35..978b871c4bc 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.24.2 + 2.24.3 3.8.0 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 15c67bb1695..d98d4b9f797 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.24.2 +set VERSION=2.24.3 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index 3538964a3c2..9910e202fab 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.24.2' + v = '1.24.3' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 6237d3fc24b..a4a0edd047c 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.24.2' + v = '1.24.3' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 2567296f04f..15edbf20fb9 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.2" +#define GRPC_OBJC_VERSION_STRING @"1.24.3" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index db16988fcda..238587ec999 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.24.2" +#define GRPC_OBJC_VERSION_STRING @"1.24.3" #define GRPC_C_VERSION_STRING @"8.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index 737028cb1b7..bf0d2a30054 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.24.2", + "version": "1.24.3", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index a58e5a2a67d..5b3cf9342c6 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.24.2" +#define PHP_GRPC_VERSION "1.24.3" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 63809489efa..c5f1cba535e 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.24.2""" +__version__ = """1.24.3""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index e72c465e392..04ad6ea3694 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index c2945d4b362..322e38e08d4 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index e105ce583b7..45a8bf44f3f 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 880ddbfaf62..9cae0c7786a 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index 809dcfa90e1..38d552c3a7d 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index d88211f92d1..7315ad0564b 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index be0ae55d30f..a5e5eff9b70 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index b1760a0ba39..71fb6806214 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.24.2' + VERSION = '1.24.3' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 863aa765e62..0965d960f5f 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.24.2' + VERSION = '1.24.3' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index f478434ee4d..306b71bdb7d 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.24.2' +VERSION = '1.24.3' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 63d6a1b1db8..0d16835037f 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.2 +PROJECT_NUMBER = 1.24.3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index dfd6c037f9d..c5610c756fe 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.24.2 +PROJECT_NUMBER = 1.24.3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 35b46aa07007f46a65020fab1db4ff8978ea77bd Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 15 Oct 2019 15:59:06 -0700 Subject: [PATCH 24/27] Release 3.8 wheels --- setup.py | 2 ++ src/python/grpcio_channelz/setup.py | 2 ++ src/python/grpcio_health_checking/setup.py | 2 ++ src/python/grpcio_reflection/setup.py | 2 ++ src/python/grpcio_status/setup.py | 1 + tools/run_tests/artifacts/artifact_targets.py | 10 ++++++++++ 6 files changed, 19 insertions(+) diff --git a/setup.py b/setup.py index 1c668999234..c3e95f861f3 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,8 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_channelz/setup.py b/src/python/grpcio_channelz/setup.py index cc03809dda8..d706e18529e 100644 --- a/src/python/grpcio_channelz/setup.py +++ b/src/python/grpcio_channelz/setup.py @@ -53,6 +53,8 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index dc2a69c1f43..efa781deff3 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -52,6 +52,8 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 3fbcfda3229..efe7a13310e 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -53,6 +53,8 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_status/setup.py b/src/python/grpcio_status/setup.py index 06d5dcfa8aa..5c60e78d592 100644 --- a/src/python/grpcio_status/setup.py +++ b/src/python/grpcio_status/setup.py @@ -53,6 +53,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: Apache Software License', ] diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 762c1a32399..23599443f70 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -371,41 +371,51 @@ def targets(): PythonArtifact('manylinux1', 'x86', 'cp35-cp35m'), PythonArtifact('manylinux1', 'x86', 'cp36-cp36m'), PythonArtifact('manylinux1', 'x86', 'cp37-cp37m'), + PythonArtifact('manylinux1', 'x86', 'cp38-cp38'), PythonArtifact('linux_extra', 'armv7', '2.7'), PythonArtifact('linux_extra', 'armv7', '3.4'), PythonArtifact('linux_extra', 'armv7', '3.5'), PythonArtifact('linux_extra', 'armv7', '3.6'), + PythonArtifact('linux_extra', 'armv7', '3.7'), + PythonArtifact('linux_extra', 'armv7', '3.8'), PythonArtifact('linux_extra', 'armv6', '2.7'), PythonArtifact('linux_extra', 'armv6', '3.4'), PythonArtifact('linux_extra', 'armv6', '3.5'), PythonArtifact('linux_extra', 'armv6', '3.6'), + PythonArtifact('linux_extra', 'armv6', '3.7'), + PythonArtifact('linux_extra', 'armv6', '3.8'), PythonArtifact('manylinux1', 'x64', 'cp27-cp27m'), PythonArtifact('manylinux1', 'x64', 'cp27-cp27mu'), PythonArtifact('manylinux1', 'x64', 'cp34-cp34m'), PythonArtifact('manylinux1', 'x64', 'cp35-cp35m'), PythonArtifact('manylinux1', 'x64', 'cp36-cp36m'), PythonArtifact('manylinux1', 'x64', 'cp37-cp37m'), + PythonArtifact('manylinux1', 'x64', 'cp38-cp38'), PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), PythonArtifact('manylinux2010', 'x64', 'cp34-cp34m'), PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), + PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'), PythonArtifact('macos', 'x64', 'python2.7'), PythonArtifact('macos', 'x64', 'python3.4'), PythonArtifact('macos', 'x64', 'python3.5'), PythonArtifact('macos', 'x64', 'python3.6'), PythonArtifact('macos', 'x64', 'python3.7'), + PythonArtifact('macos', 'x64', 'python3.8'), PythonArtifact('windows', 'x86', 'Python27_32bits'), PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), PythonArtifact('windows', 'x86', 'Python37_32bits'), + PythonArtifact('windows', 'x86', 'Python38_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python36'), PythonArtifact('windows', 'x64', 'Python37'), + PythonArtifact('windows', 'x64', 'Python38'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact('linux', 'x64') From cb9ab57de8a8bb2841d608da0c3f679b3f156a4e Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 15 Oct 2019 17:49:08 -0700 Subject: [PATCH 25/27] Remove macOS and Windows 3.8 artifacts --- tools/run_tests/artifacts/artifact_targets.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 23599443f70..1ab64f5d65c 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -403,19 +403,22 @@ def targets(): PythonArtifact('macos', 'x64', 'python3.5'), PythonArtifact('macos', 'x64', 'python3.6'), PythonArtifact('macos', 'x64', 'python3.7'), - PythonArtifact('macos', 'x64', 'python3.8'), + # TODO(https://github.com/grpc/grpc/issues/20615) Enable this artifact + # PythonArtifact('macos', 'x64', 'python3.8'), PythonArtifact('windows', 'x86', 'Python27_32bits'), PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), PythonArtifact('windows', 'x86', 'Python37_32bits'), - PythonArtifact('windows', 'x86', 'Python38_32bits'), + # TODO(https://github.com/grpc/grpc/issues/20615) Enable this artifact + # PythonArtifact('windows', 'x86', 'Python38_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python36'), PythonArtifact('windows', 'x64', 'Python37'), - PythonArtifact('windows', 'x64', 'Python38'), + # TODO(https://github.com/grpc/grpc/issues/20615) Enable this artifact + # PythonArtifact('windows', 'x64', 'Python38'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact('linux', 'x64') From d2eaaf16f9b04dba6d3ca746bfc31060aef871b4 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 18 Oct 2019 10:37:34 -0700 Subject: [PATCH 26/27] Add manylinux2010_x86 wheels into artifacts --- tools/run_tests/artifacts/artifact_targets.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 1ab64f5d65c..e41c2a0db30 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -372,18 +372,23 @@ def targets(): PythonArtifact('manylinux1', 'x86', 'cp36-cp36m'), PythonArtifact('manylinux1', 'x86', 'cp37-cp37m'), PythonArtifact('manylinux1', 'x86', 'cp38-cp38'), + PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), + PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), + PythonArtifact('manylinux2010', 'x86', 'cp34-cp34m'), + PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), + PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), + PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), + PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), PythonArtifact('linux_extra', 'armv7', '2.7'), PythonArtifact('linux_extra', 'armv7', '3.4'), PythonArtifact('linux_extra', 'armv7', '3.5'), PythonArtifact('linux_extra', 'armv7', '3.6'), PythonArtifact('linux_extra', 'armv7', '3.7'), - PythonArtifact('linux_extra', 'armv7', '3.8'), PythonArtifact('linux_extra', 'armv6', '2.7'), PythonArtifact('linux_extra', 'armv6', '3.4'), PythonArtifact('linux_extra', 'armv6', '3.5'), PythonArtifact('linux_extra', 'armv6', '3.6'), PythonArtifact('linux_extra', 'armv6', '3.7'), - PythonArtifact('linux_extra', 'armv6', '3.8'), PythonArtifact('manylinux1', 'x64', 'cp27-cp27m'), PythonArtifact('manylinux1', 'x64', 'cp27-cp27mu'), PythonArtifact('manylinux1', 'x64', 'cp34-cp34m'), From 14848834ddb5efcf3fdff4b1ed2f41d193b40c00 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 18 Oct 2019 15:58:39 -0700 Subject: [PATCH 27/27] Remove 3.7 ARM wheels --- tools/run_tests/artifacts/artifact_targets.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index e41c2a0db30..a240c3f6207 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -383,12 +383,10 @@ def targets(): PythonArtifact('linux_extra', 'armv7', '3.4'), PythonArtifact('linux_extra', 'armv7', '3.5'), PythonArtifact('linux_extra', 'armv7', '3.6'), - PythonArtifact('linux_extra', 'armv7', '3.7'), PythonArtifact('linux_extra', 'armv6', '2.7'), PythonArtifact('linux_extra', 'armv6', '3.4'), PythonArtifact('linux_extra', 'armv6', '3.5'), PythonArtifact('linux_extra', 'armv6', '3.6'), - PythonArtifact('linux_extra', 'armv6', '3.7'), PythonArtifact('manylinux1', 'x64', 'cp27-cp27m'), PythonArtifact('manylinux1', 'x64', 'cp27-cp27mu'), PythonArtifact('manylinux1', 'x64', 'cp34-cp34m'),