diff --git a/BUILD.bazel b/BUILD.bazel index 1a305e7f19..a14476a553 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -266,7 +266,7 @@ alias( alias( name = "well_known_types_py_pb2", actual = "//python:well_known_types_py_pb2", - visibility = ["@upb//:__subpackages__"], + visibility = ["//visibility:public"], ) alias( diff --git a/CHANGES.txt b/CHANGES.txt index a8e7fd19a9..a7d74a7eee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,14 +6,26 @@ * Add C++20 keywords guarded by PROTOBUF_FUTURE_CPP20_KEYWORDS * Fixed crash in ThreadLocalStorage for pre-C++17 compilers on 32-bit ARM. * Clarified that JSON API non-OK statuses are not a stable API. + * Added a default implementation of MessageDifferencer::Reporter methods. + * proto2::MapPair is now an alias to std::pair. + * Hide C++ RepeatedField::UnsafeArenaSwap Kotlin * Kotlin generated code comments now use kdoc format instead of javadoc. * Escape keywords in package names in proto generated code + * Add Kotlin enum int value getters and setters Java * Performance improvement for repeated use of FieldMaskUtil#merge by caching constructed FieldMaskTrees. + * Optimized Java proto serialization gencode for protos having many extension ranges with few fields in between. + + Python + * Changes ordering of printed fields in .pyi files from lexicographic to the same ordering found in the proto descriptor. + + Compiler + * Print full path name of source .proto file on error + 2022-07-25 version 21.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04cb3303ac..2ed5ca1141 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,9 +57,11 @@ endif() option(protobuf_BUILD_TESTS "Build tests" ON) option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF) option(protobuf_BUILD_EXAMPLES "Build examples" OFF) +option(protobuf_BUILD_PROTOBUF_BINARIES "Build protobuf libraries and protoc compiler" ON) option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON) option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF) option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF) +option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "") if (BUILD_SHARED_LIBS) set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON) else (BUILD_SHARED_LIBS) @@ -81,6 +83,9 @@ include(${protobuf_SOURCE_DIR}/cmake/protobuf-options.cmake) if (protobuf_BUILD_PROTOC_BINARIES OR protobuf_BUILD_TESTS) set(protobuf_BUILD_LIBPROTOC ON) endif () +if (NOT protobuf_BUILD_PROTOBUF_BINARIES) + set(protobuf_INSTALL OFF) +endif() # Path to main configure script set(protobuf_CONFIGURE_SCRIPT "${protobuf_SOURCE_DIR}/configure.ac") @@ -242,12 +247,12 @@ endif (protobuf_BUILD_SHARED_LIBS) if (MSVC) if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # Build with multiple processes - add_definitions(/MP) + add_compile_options(/MP) endif() # Set source file and execution character sets to UTF-8 - add_definitions(/utf-8) + add_compile_options(/utf-8) # MSVC warning suppressions - add_definitions( + add_compile_options( /wd4065 # switch statement contains 'default' but no 'case' labels /wd4244 # 'conversion' conversion from 'type1' to 'type2', possible loss of data /wd4251 # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' @@ -262,23 +267,16 @@ if (MSVC) /wd4996 # The compiler encountered a deprecated declaration. ) # Allow big object - add_definitions(/bigobj) + add_compile_options(/bigobj) string(REPLACE "/" "\\" PROTOBUF_SOURCE_WIN32_PATH ${protobuf_SOURCE_DIR}) string(REPLACE "/" "\\" PROTOBUF_BINARY_WIN32_PATH ${protobuf_BINARY_DIR}) string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") - configure_file(${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in extract_includes.bat) # Suppress linker warnings about files with no symbols defined. - set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /ignore:4221") + string(APPEND CMAKE_STATIC_LINKER_FLAGS " /ignore:4221") - if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - # Configure Resource Compiler - enable_language(RC) - # use English language (0x409) in resource compiler - set(rc_flags "/l0x409") - # fix rc.exe invocations because of usage of add_definitions() - set(CMAKE_RC_COMPILE_OBJECT " ${rc_flags} /fo ") - endif() + # use English language (0x409) in resource compiler + string(APPEND CMAKE_RC_FLAGS " -l0x409") # Generate the version.rc file used elsewhere. configure_file(${protobuf_SOURCE_DIR}/cmake/version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY) @@ -304,28 +302,49 @@ if (protobuf_UNICODE) add_definitions(-DUNICODE -D_UNICODE) endif (protobuf_UNICODE) -include(${protobuf_SOURCE_DIR}/cmake/libprotobuf-lite.cmake) -include(${protobuf_SOURCE_DIR}/cmake/libprotobuf.cmake) -if (protobuf_BUILD_LIBPROTOC) - include(${protobuf_SOURCE_DIR}/cmake/libprotoc.cmake) -endif (protobuf_BUILD_LIBPROTOC) -if (protobuf_BUILD_PROTOC_BINARIES) - include(${protobuf_SOURCE_DIR}/cmake/protoc.cmake) - if (NOT DEFINED protobuf_PROTOC_EXE) - set(protobuf_PROTOC_EXE protoc) - endif (NOT DEFINED protobuf_PROTOC_EXE) -endif (protobuf_BUILD_PROTOC_BINARIES) +if (protobuf_BUILD_PROTOBUF_BINARIES) + include(${protobuf_SOURCE_DIR}/cmake/libprotobuf-lite.cmake) + if (NOT DEFINED protobuf_LIB_PROTOBUF_LITE) + set(protobuf_LIB_PROTOBUF_LITE libprotobuf-lite) + endif () + include(${protobuf_SOURCE_DIR}/cmake/libprotobuf.cmake) + if (NOT DEFINED protobuf_LIB_PROTOBUF) + set(protobuf_LIB_PROTOBUF libprotobuf) + endif () + if (protobuf_BUILD_LIBPROTOC) + include(${protobuf_SOURCE_DIR}/cmake/libprotoc.cmake) + if (NOT DEFINED protobuf_LIB_PROTOC) + set(protobuf_LIB_PROTOC libprotoc) + endif () + endif () + if (protobuf_BUILD_PROTOC_BINARIES) + include(${protobuf_SOURCE_DIR}/cmake/protoc.cmake) + if (NOT DEFINED protobuf_PROTOC_EXE) + set(protobuf_PROTOC_EXE protoc) + endif () + endif () +else () + find_package(protobuf) + if (protobuf_FOUND) + set(protobuf_PROTOC_EXE protobuf::protoc) + set(protobuf_LIB_PROTOC protobuf::libprotoc) + set(protobuf_LIB_PROTOBUF protobuf::libprotobuf) + set(protobuf_LIB_PROTOBUF_LITE protobuf::libprotobuf-lite) + message(STATUS "CMake installation of Protobuf found.") + endif () +endif () -# Ensure we have a protoc executable if we need one +# Ensure we have a protoc executable and protobuf libraries if we need one if (protobuf_BUILD_TESTS OR protobuf_BUILD_CONFORMANCE OR protobuf_BUILD_EXAMPLES) if (NOT DEFINED protobuf_PROTOC_EXE) - find_program(protobuf_PROTOC_EXE protoc) - if (NOT protobuf_PROTOC_EXE) - message(FATAL "Build requires 'protoc' but binary not found and not building protoc.") - endif () + find_program(protobuf_PROTOC_EXE protoc REQUIRED) + message(STATUS "Found system ${protobuf_PROTOC_EXE}.") endif () if(protobuf_VERBOSE) message(STATUS "Using protoc : ${protobuf_PROTOC_EXE}") + message(STATUS "Using libprotobuf : ${protobuf_LIB_PROTOBUF}") + message(STATUS "Using libprotobuf-lite : ${protobuf_LIB_PROTOBUF_LITE}") + message(STATUS "Using libprotoc : ${protobuf_LIB_PROTOC}") endif(protobuf_VERBOSE) endif () diff --git a/Makefile.am b/Makefile.am index d35bcaf27e..acc72c27ef 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,11 +131,13 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf.Test/GeneratedMessageTest.Proto2.cs \ csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj \ csharp/src/Google.Protobuf.Test/IssuesTest.cs \ + csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs \ csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs \ csharp/src/Google.Protobuf.Test/JsonParserTest.cs \ csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs \ csharp/src/Google.Protobuf.Test/LegacyGeneratedCodeTest.cs \ csharp/src/Google.Protobuf.Test/MessageParsingHelpers.cs \ + csharp/src/Google.Protobuf.Test/ParsingPrimitivesTest.cs \ csharp/src/Google.Protobuf.Test/Proto3OptionalTest.cs \ csharp/src/Google.Protobuf.Test/ReadOnlySequenceFactory.cs \ csharp/src/Google.Protobuf.Test/RefStructCompatibilityTest.cs \ @@ -174,6 +176,7 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf.Test/WellKnownTypes/FieldMaskTest.cs \ csharp/src/Google.Protobuf.Test/WellKnownTypes/TimestampTest.cs \ csharp/src/Google.Protobuf.Test/WellKnownTypes/WrappersTest.cs \ + csharp/src/Google.Protobuf.Test/WritingPrimitivesTest.cs \ csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs \ csharp/src/Google.Protobuf.Test/testprotos.pb \ csharp/src/Google.Protobuf.sln \ @@ -958,6 +961,8 @@ php_EXTRA_DIST= \ php/tests/EncodeDecodeTest.php \ php/tests/force_c_ext.php \ php/tests/gdb_test.sh \ + php/tests/generated_previous/GPBMetadata/ProtoPrevious/TestPreviouslyUnreservedMessage.php \ + php/tests/generated_previous/Previous/readonly.php \ php/tests/GeneratedClassTest.php \ php/tests/GeneratedPhpdocTest.php \ php/tests/GeneratedServiceTest.php \ @@ -967,6 +972,7 @@ php_EXTRA_DIST= \ php/tests/multirequest.php \ php/tests/multirequest.sh \ php/tests/PhpImplementationTest.php \ + php/tests/PreviouslyGeneratedClassTest.php \ php/tests/proto/empty/echo.proto \ php/tests/proto/test.proto \ php/tests/proto/test_descriptors.proto \ @@ -985,6 +991,7 @@ php_EXTRA_DIST= \ php/tests/proto/test_service.proto \ php/tests/proto/test_service_namespace.proto \ php/tests/proto/test_wrapper_type_setters.proto \ + php/tests/proto_previous/test_previously_unreserved_message.proto \ php/tests/test_base.php \ php/tests/test_util.php \ php/tests/valgrind.supp \ @@ -1214,7 +1221,6 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ cmake/README.md \ cmake/conformance.cmake \ cmake/examples.cmake \ - cmake/extract_includes.bat.in \ cmake/install.cmake \ cmake/libprotobuf-lite.cmake \ cmake/libprotobuf.cmake \ diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index c8a516bb05..858a861a5b 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.21.2' + s.version = '3.21.4' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/cmake/README.md b/cmake/README.md index ce3e680234..4a4259e45f 100644 --- a/cmake/README.md +++ b/cmake/README.md @@ -1,15 +1,19 @@ -This directory contains *CMake* files that can be used to build protobuf -with *MSVC* on *Windows*. You can build the project from *Command Prompt* -and using an *Visual Studio* IDE. +This directory contains *CMake* files that can be used to build protobuf. -You need to have [CMake](http://www.cmake.org), [Visual Studio](https://www.visualstudio.com) -and optionally [Git](http://git-scm.com) installed on your computer before proceeding. +You need to have [CMake](http://www.cmake.org) and +[Git](http://git-scm.com) installed on your computer before proceeding. -Most of the instructions will be given to the *Сommand Prompt*, but the same -actions can be performed using appropriate GUI tools. +Most of the instructions will be given using CMake's command-line interface, but +the same actions can be performed using appropriate GUI tools. -Environment Setup -================= +# Windows Builds + +On Windows, you can build the project from *Command Prompt* and using an +*Visual Studio* IDE. You will also need to have +[Visual Studio](https://www.visualstudio.com) installed on your computer before +proceeding. + +## Environment Setup Open the appropriate *Command Prompt* from the *Start* menu. @@ -42,8 +46,7 @@ Optionally, you will want to download [ninja](https://ninja-build.org/) and add Good. Now you are ready to continue. -Getting Sources -=============== +## Getting Sources You can get the latest stable source packages from the release page: @@ -76,8 +79,7 @@ C:\Path\to\src\protobuf> git submodule update --init --recursive Good. Now you are ready for *CMake* configuration. -CMake Configuration -=================== +## CMake Configuration *CMake* supports a lot of different [generators](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html) @@ -137,8 +139,7 @@ The *Visual Studio* generator is multi-configuration: it will generate a single It will generate *Visual Studio* solution file *protobuf.sln* in current directory. -Unit Tests ----------- +### Unit Tests Unit tests are being built along with the rest of protobuf. The unit tests require Google Mock (now a part of Google Test). @@ -178,8 +179,7 @@ For example: -Dprotobuf_BUILD_TESTS=OFF ^ C:\Path\to\src\protobuf -Compiling -========= +## Compiling The standard way to compile a *CMake* project is `cmake --build `. @@ -206,8 +206,7 @@ If you prefer to use the IDE: And wait for the compilation to finish. -Testing -======= +## Testing To run unit-tests, first you must compile protobuf as described above. Then run: @@ -259,8 +258,7 @@ Note that the tests must be run from the source folder. If all tests are passed, safely continue. -Installing -========== +## Installing To install protobuf to the *install* folder you've specified in the configuration step, you need to build the `install` target: @@ -292,8 +290,7 @@ compiling a debug build of your application, you may need to link against a debug build of libprotobufd.lib with "d" postfix. Similarly, release builds should link against release libprotobuf.lib library. -DLLs vs. static linking -======================= +## DLLs vs. static linking Static linking is now the default for the Protocol Buffer libraries. Due to issues with Win32's use of a separate heap for each DLL, as well as binary @@ -318,8 +315,7 @@ recommend that you do NOT expose protocol buffer objects in your library's public interface, and that you statically link protocol buffers into your library. -ZLib support -============ +## ZLib support If you want to include GzipInputStream and GzipOutputStream (google/protobuf/io/gzip_stream.h) in libprotobuf, you will need to do a few @@ -369,8 +365,7 @@ If you already have ZLIB library and headers at some other location on your syst Build and testing protobuf as usual. -Notes on Compiler Warnings -========================== +## Notes on Compiler Warnings The following warnings have been disabled while building the protobuf libraries and compiler. You may have to disable some of them in your own project as @@ -397,3 +392,23 @@ unique, so there should be no problem with this, but MSVC prints warning nevertheless. So, we disable it. Unfortunately, this warning will also be produced when compiling code which merely uses protocol buffers, meaning you may have to disable it in your code too. + +# Linux Builds + +Building with CMake works very similarly on Linux. Instead of Visual Studio, +you will need to have gcc or clang installed to handle the C++ builds. CMake +will generate Makefiles by default, but can also be configured to use Ninja. To +build Protobuf, you will need to run (from the source directory): + + cmake . + cmake --build . --parallel 10 + +Protobuf can be tested and installed with CMake: + + ctest --verbose + sudo cmake --install . + +or directly with the generated Makefiles: + + make VERBOSE=1 test + sudo make install diff --git a/cmake/conformance.cmake b/cmake/conformance.cmake index d6c435ac33..338b2757a5 100644 --- a/cmake/conformance.cmake +++ b/cmake/conformance.cmake @@ -24,6 +24,9 @@ add_executable(conformance_test_runner ${protobuf_SOURCE_DIR}/conformance/conformance.pb.cc ${protobuf_SOURCE_DIR}/conformance/conformance_test.cc ${protobuf_SOURCE_DIR}/conformance/conformance_test_runner.cc + ${protobuf_SOURCE_DIR}/conformance/conformance_test_main.cc + ${protobuf_SOURCE_DIR}/conformance/text_format_conformance_suite.cc + ${protobuf_SOURCE_DIR}/conformance/text_format_conformance_suite.h ${protobuf_SOURCE_DIR}/conformance/third_party/jsoncpp/json.h ${protobuf_SOURCE_DIR}/conformance/third_party/jsoncpp/jsoncpp.cpp ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc @@ -45,5 +48,13 @@ target_include_directories( conformance_cpp PUBLIC ${protobuf_SOURCE_DIR}/conformance) -target_link_libraries(conformance_test_runner libprotobuf) -target_link_libraries(conformance_cpp libprotobuf) +target_link_libraries(conformance_test_runner ${protobuf_LIB_PROTOBUF}) +target_link_libraries(conformance_cpp ${protobuf_LIB_PROTOBUF}) + +add_test(NAME conformance_cpp_test + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/conformance_test_runner + --failure_list ${protobuf_SOURCE_DIR}/conformance/failure_list_cpp.txt + --text_format_failure_list ${protobuf_SOURCE_DIR}/conformance/text_format_failure_list_cpp.txt + --output_dir ${protobuf_TEST_XML_OUTDIR} + ${CMAKE_CURRENT_BINARY_DIR}/conformance_cpp + DEPENDS conformance_test_runner conformance_cpp) diff --git a/cmake/examples.cmake b/cmake/examples.cmake index 3b83d2b4cf..07cfad2b9f 100644 --- a/cmake/examples.cmake +++ b/cmake/examples.cmake @@ -30,7 +30,7 @@ endfunction() # Add examples as an external project. # sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets. add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}") -add_dependencies(examples libprotobuf protoc) +add_dependencies(examples ${protobuf_LIB_PROTOBUF} ${protobuf_PROTOC_EXE}) option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF) mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST) @@ -42,7 +42,7 @@ if(protobuf_BUILD_EXAMPLES_MULTITEST) "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}" "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE" ) - add_dependencies(examples-legacy libprotobuf protoc) + add_dependencies(examples-legacy ${protobuf_LIB_PROTOBUF} ${protobuf_PROTOC_EXE}) #Build using the installed library. add_examples_build(examples-installed diff --git a/cmake/install.cmake b/cmake/install.cmake index 2da8170544..cf24e307c1 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -43,25 +43,23 @@ endif (protobuf_BUILD_PROTOC_BINARIES) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc ${CMAKE_CURRENT_BINARY_DIR}/protobuf-lite.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") -file(STRINGS ${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in _extract_strings - REGEX "^copy") -foreach(_extract_string ${_extract_strings}) - string(REGEX REPLACE "^.* .+ include\\\\(.+)$" "\\1" - _header ${_extract_string}) - string(REPLACE "\\" "/" _header ${_header}) +include(${protobuf_SOURCE_DIR}/src/file_lists.cmake) +set(protobuf_HEADERS + ${libprotobuf_hdrs} + ${libprotoc_hdrs} + ${wkt_protos_files} + ${descriptor_proto_proto_srcs} + ${plugin_proto_proto_srcs} +) +foreach(_header ${protobuf_HEADERS}) + string(REPLACE "${protobuf_SOURCE_DIR}/src" "" _header ${_header}) get_filename_component(_extract_from "${protobuf_SOURCE_DIR}/src/${_header}" ABSOLUTE) get_filename_component(_extract_name ${_header} NAME) get_filename_component(_extract_to "${CMAKE_INSTALL_INCLUDEDIR}/${_header}" DIRECTORY) - if(EXISTS "${_extract_from}") - install(FILES "${_extract_from}" - DESTINATION "${_extract_to}" - COMPONENT protobuf-headers - RENAME "${_extract_name}") - else() - message(AUTHOR_WARNING "The file \"${_extract_from}\" is listed in " - "\"${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in\" " - "but there not exists. The file will not be installed.") - endif() + install(FILES "${_extract_from}" + DESTINATION "${_extract_to}" + COMPONENT protobuf-headers + RENAME "${_extract_name}") endforeach() # Internal function for parsing auto tools scripts diff --git a/cmake/tests.cmake b/cmake/tests.cmake index be1d429ab8..6a1d1c09cc 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -1,6 +1,7 @@ option(protobuf_USE_EXTERNAL_GTEST "Use external Google Test (i.e. not the one in third_party/googletest)" OFF) -option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "") +option(protobuf_REMOVE_INSTALLED_HEADERS + "Remove local headers so that installed ones are used instead" OFF) option(protobuf_ABSOLUTE_TEST_PLUGIN_PATH "Using absolute test_plugin path in tests" ON) @@ -77,10 +78,9 @@ endforeach(proto_file) add_library(protobuf-lite-test-common STATIC ${lite_test_util_srcs} ${lite_test_proto_files}) -target_link_libraries(protobuf-lite-test-common libprotobuf-lite GTest::gmock) +target_link_libraries(protobuf-lite-test-common ${protobuf_LIB_PROTOBUF_LITE} GTest::gmock) set(common_test_files - ${lite_test_util_srcs} ${test_util_hdrs} ${test_util_srcs} ${mock_code_generator_srcs} @@ -89,7 +89,7 @@ set(common_test_files add_library(protobuf-test-common STATIC ${common_test_files} ${tests_proto_files}) -target_link_libraries(protobuf-test-common libprotobuf GTest::gmock) +target_link_libraries(protobuf-test-common ${protobuf_LIB_PROTOBUF} GTest::gmock) set(tests_files ${protobuf_test_files} @@ -130,7 +130,7 @@ if (MSVC) /wd4146 # unary minus operator applied to unsigned type, result still unsigned ) endif() -target_link_libraries(tests protobuf-lite-test-common protobuf-test-common libprotoc libprotobuf GTest::gmock_main) +target_link_libraries(tests protobuf-lite-test-common protobuf-test-common ${protobuf_LIB_PROTOC} ${protobuf_LIB_PROTOBUF} GTest::gmock_main) set(test_plugin_files ${test_plugin_files} @@ -139,19 +139,50 @@ set(test_plugin_files ) add_executable(test_plugin ${test_plugin_files}) -target_link_libraries(test_plugin libprotoc libprotobuf GTest::gmock) +target_link_libraries(test_plugin ${protobuf_LIB_PROTOC} ${protobuf_LIB_PROTOBUF} GTest::gmock) add_executable(lite-test ${protobuf_lite_test_files}) -target_link_libraries(lite-test protobuf-lite-test-common libprotobuf-lite GTest::gmock_main) +target_link_libraries(lite-test protobuf-lite-test-common ${protobuf_LIB_PROTOBUF_LITE} GTest::gmock_main) add_test(NAME lite-test COMMAND lite-test ${protobuf_GTEST_ARGS}) add_custom_target(check COMMAND tests - DEPENDS tests test_plugin + DEPENDS tests lite-test test_plugin WORKING_DIRECTORY ${protobuf_SOURCE_DIR}) add_test(NAME check - COMMAND tests ${protobuf_GTEST_ARGS} - WORKING_DIRECTORY "${protobuf_SOURCE_DIR}") + COMMAND tests ${protobuf_GTEST_ARGS}) + +# For test purposes, remove headers that should already be installed. This +# prevents accidental conflicts and also version skew (since local headers take +# precedence over installed headers). +add_custom_target(save-installed-headers) +add_custom_target(remove-installed-headers) +add_custom_target(restore-installed-headers) + +# Explicitly skip the bootstrapping headers as it's directly used in tests +set(_installed_hdrs ${libprotobuf_hdrs} ${libprotoc_hdrs}) +list(REMOVE_ITEM _installed_hdrs + "${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.pb.h" + "${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.pb.h") + +foreach(_hdr ${_installed_hdrs}) + string(REPLACE "${protobuf_SOURCE_DIR}/src" "" _file ${_hdr}) + set(_tmp_file "${CMAKE_BINARY_DIR}/tmp-install-test/${_file}") + add_custom_command(TARGET remove-installed-headers PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E remove -f "${_hdr}") + add_custom_command(TARGET save-installed-headers PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy "${_hdr}" "${_tmp_file}" || true) + add_custom_command(TARGET restore-installed-headers PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy "${_tmp_file}" "${_hdr}") +endforeach() + +add_dependencies(remove-installed-headers save-installed-headers) +if(protobuf_REMOVE_INSTALLED_HEADERS) + add_dependencies(protobuf-lite-test-common remove-installed-headers) + add_dependencies(protobuf-test-common remove-installed-headers) +endif() diff --git a/configure.ac b/configure.ac index c55a6a2c2f..64ddb3dc84 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.21.2],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.21.4],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs index 57e59a9ac3..d3284a40f6 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs @@ -580,23 +580,23 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapStringString_.Add(other.mapStringString_); - mapInt32Bytes_.Add(other.mapInt32Bytes_); - mapInt32Enum_.Add(other.mapInt32Enum_); - mapInt32ForeignMessage_.Add(other.mapInt32ForeignMessage_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapStringString_.MergeFrom(other.mapStringString_); + mapInt32Bytes_.MergeFrom(other.mapInt32Bytes_); + mapInt32Enum_.MergeFrom(other.mapInt32Enum_); + mapInt32ForeignMessage_.MergeFrom(other.mapInt32ForeignMessage_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1100,7 +1100,7 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - mapInt32Message_.Add(other.mapInt32Message_); + mapInt32Message_.MergeFrom(other.mapInt32Message_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1298,8 +1298,8 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - map1_.Add(other.map1_); - map2_.Add(other.map2_); + map1_.MergeFrom(other.map1_); + map2_.MergeFrom(other.map2_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1723,21 +1723,21 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapInt32Enum_.Add(other.mapInt32Enum_); - mapInt32ForeignMessage_.Add(other.mapInt32ForeignMessage_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapInt32Enum_.MergeFrom(other.mapInt32Enum_); + mapInt32ForeignMessage_.MergeFrom(other.mapInt32ForeignMessage_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -2031,7 +2031,7 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - type_.Add(other.type_); + type_.MergeFrom(other.type_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -2224,7 +2224,7 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - entry_.Add(other.entry_); + entry_.MergeFrom(other.entry_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs index 08b1aaf80c..d5734552c0 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs @@ -4344,25 +4344,25 @@ namespace ProtobufTestMessages.Proto2 { unpackedDouble_.Add(other.unpackedDouble_); unpackedBool_.Add(other.unpackedBool_); unpackedNestedEnum_.Add(other.unpackedNestedEnum_); - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapStringString_.Add(other.mapStringString_); - mapStringBytes_.Add(other.mapStringBytes_); - mapStringNestedMessage_.Add(other.mapStringNestedMessage_); - mapStringForeignMessage_.Add(other.mapStringForeignMessage_); - mapStringNestedEnum_.Add(other.mapStringNestedEnum_); - mapStringForeignEnum_.Add(other.mapStringForeignEnum_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapStringString_.MergeFrom(other.mapStringString_); + mapStringBytes_.MergeFrom(other.mapStringBytes_); + mapStringNestedMessage_.MergeFrom(other.mapStringNestedMessage_); + mapStringForeignMessage_.MergeFrom(other.mapStringForeignMessage_); + mapStringNestedEnum_.MergeFrom(other.mapStringNestedEnum_); + mapStringForeignEnum_.MergeFrom(other.mapStringForeignEnum_); if (other.HasData) { if (!HasData) { Data = new global::ProtobufTestMessages.Proto2.TestAllTypesProto2.Types.Data(); diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs index 520216fdf5..74e2a57680 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs @@ -3753,25 +3753,25 @@ namespace ProtobufTestMessages.Proto3 { unpackedDouble_.Add(other.unpackedDouble_); unpackedBool_.Add(other.unpackedBool_); unpackedNestedEnum_.Add(other.unpackedNestedEnum_); - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapStringString_.Add(other.mapStringString_); - mapStringBytes_.Add(other.mapStringBytes_); - mapStringNestedMessage_.Add(other.mapStringNestedMessage_); - mapStringForeignMessage_.Add(other.mapStringForeignMessage_); - mapStringNestedEnum_.Add(other.mapStringNestedEnum_); - mapStringForeignEnum_.Add(other.mapStringForeignEnum_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapStringString_.MergeFrom(other.mapStringString_); + mapStringBytes_.MergeFrom(other.mapStringBytes_); + mapStringNestedMessage_.MergeFrom(other.mapStringNestedMessage_); + mapStringForeignMessage_.MergeFrom(other.mapStringForeignMessage_); + mapStringNestedEnum_.MergeFrom(other.mapStringNestedEnum_); + mapStringForeignEnum_.MergeFrom(other.mapStringForeignEnum_); if (other.optionalBoolWrapper_ != null) { if (optionalBoolWrapper_ == null || other.OptionalBoolWrapper != false) { OptionalBoolWrapper = other.OptionalBoolWrapper; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs b/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs index c1f43ce0f8..7f1aca1b92 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs @@ -24112,7 +24112,7 @@ namespace Google.Protobuf.TestProtos.Proto2 { if (other == null) { return; } - foo_.Add(other.foo_); + foo_.MergeFrom(other.foo_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -30708,7 +30708,7 @@ namespace Google.Protobuf.TestProtos.Proto2 { } OptionalGroup.MergeFrom(other.OptionalGroup); } - stringStringMap_.Add(other.stringStringMap_); + stringStringMap_.MergeFrom(other.stringStringMap_); switch (other.OneofFieldCase) { case OneofFieldOneofCase.OneofUint32: OneofUint32 = other.OneofUint32; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs index 3ec8d3556f..50b9046a3b 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs @@ -3258,24 +3258,24 @@ namespace Google.Protobuf.TestProtos { if (other == null) { return; } - anyField_.Add(other.anyField_); - apiField_.Add(other.apiField_); - durationField_.Add(other.durationField_); - emptyField_.Add(other.emptyField_); - fieldMaskField_.Add(other.fieldMaskField_); - sourceContextField_.Add(other.sourceContextField_); - structField_.Add(other.structField_); - timestampField_.Add(other.timestampField_); - typeField_.Add(other.typeField_); - doubleField_.Add(other.doubleField_); - floatField_.Add(other.floatField_); - int64Field_.Add(other.int64Field_); - uint64Field_.Add(other.uint64Field_); - int32Field_.Add(other.int32Field_); - uint32Field_.Add(other.uint32Field_); - boolField_.Add(other.boolField_); - stringField_.Add(other.stringField_); - bytesField_.Add(other.bytesField_); + anyField_.MergeFrom(other.anyField_); + apiField_.MergeFrom(other.apiField_); + durationField_.MergeFrom(other.durationField_); + emptyField_.MergeFrom(other.emptyField_); + fieldMaskField_.MergeFrom(other.fieldMaskField_); + sourceContextField_.MergeFrom(other.sourceContextField_); + structField_.MergeFrom(other.structField_); + timestampField_.MergeFrom(other.timestampField_); + typeField_.MergeFrom(other.typeField_); + doubleField_.MergeFrom(other.doubleField_); + floatField_.MergeFrom(other.floatField_); + int64Field_.MergeFrom(other.int64Field_); + uint64Field_.MergeFrom(other.uint64Field_); + int32Field_.MergeFrom(other.int32Field_); + uint32Field_.MergeFrom(other.uint32Field_); + boolField_.MergeFrom(other.boolField_); + stringField_.MergeFrom(other.stringField_); + bytesField_.MergeFrom(other.bytesField_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } diff --git a/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs b/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs index 8387291956..17c5249140 100644 --- a/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs +++ b/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs @@ -36,6 +36,7 @@ using Google.Protobuf.TestProtos; using NUnit.Framework; using System.Linq; using Google.Protobuf.WellKnownTypes; +using Google.Protobuf.Collections; namespace Google.Protobuf { @@ -795,5 +796,44 @@ namespace Google.Protobuf EqualityTester.AssertInequality(message1, message2); EqualityTester.AssertEquality(message1, message3); } + + [Test] + [TestCase(false)] + [TestCase(true)] + public void MapFieldMerging(bool direct) + { + var message1 = new TestMap + { + MapStringString = + { + { "x1", "y1" }, + { "common", "message1" } + } + }; + var message2 = new TestMap + { + MapStringString = + { + { "x2", "y2" }, + { "common", "message2" } + } + }; + if (direct) + { + message1.MergeFrom(message2); + } + else + { + message1.MergeFrom(message2.ToByteArray()); + } + + var expected = new MapField + { + { "x1", "y1" }, + { "x2", "y2" }, + { "common", "message2" } + }; + Assert.AreEqual(expected, message1.MapStringString); + } } } \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/Collections/MapField.cs b/csharp/src/Google.Protobuf/Collections/MapField.cs index f0124ee12b..09afb75cdb 100644 --- a/csharp/src/Google.Protobuf/Collections/MapField.cs +++ b/csharp/src/Google.Protobuf/Collections/MapField.cs @@ -237,6 +237,21 @@ namespace Google.Protobuf.Collections } } + /// + /// Adds the specified entries to the map, replacing any existing entries with the same keys. + /// The keys and values are not automatically cloned. + /// + /// This method primarily exists to be called from MergeFrom methods in generated classes for messages. + /// The entries to add to the map. + public void MergeFrom(IDictionary entries) + { + ProtoPreconditions.CheckNotNull(entries, nameof(entries)); + foreach (var pair in entries) + { + this[pair.Key] = pair.Value; + } + } + /// /// Returns an enumerator that iterates through the collection. /// diff --git a/csharp/src/Google.Protobuf/MessageParser.cs b/csharp/src/Google.Protobuf/MessageParser.cs index 66907d46fa..57102920ef 100644 --- a/csharp/src/Google.Protobuf/MessageParser.cs +++ b/csharp/src/Google.Protobuf/MessageParser.cs @@ -171,6 +171,10 @@ namespace Google.Protobuf /// /// Parses a message from the given JSON. /// + /// This method always uses the default JSON parser; it is not affected by . + /// To ignore unknown fields when parsing JSON, create a using a + /// with set to true and call directly. + /// /// The JSON to parse. /// The parsed message. /// The JSON does not comply with RFC 7159 @@ -203,6 +207,9 @@ namespace Google.Protobuf /// /// Creates a new message parser which optionally discards unknown fields when parsing. /// + /// Note that this does not affect the behavior of + /// at all. To ignore unknown fields when parsing JSON, create a using a + /// with set to true and call directly. /// Whether or not to discard unknown fields when parsing. /// A newly configured message parser. public MessageParser WithDiscardUnknownFields(bool discardUnknownFields) => diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs index 8c1eec53b4..aa25686dee 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs @@ -212,7 +212,7 @@ namespace Google.Protobuf.WellKnownTypes { if (other == null) { return; } - fields_.Add(other.fields_); + fields_.MergeFrom(other.fields_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } diff --git a/java/core/src/main/java/com/google/protobuf/BinaryWriter.java b/java/core/src/main/java/com/google/protobuf/BinaryWriter.java index cf394e3371..66cf51d24c 100644 --- a/java/core/src/main/java/com/google/protobuf/BinaryWriter.java +++ b/java/core/src/main/java/com/google/protobuf/BinaryWriter.java @@ -209,7 +209,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeInt32List_Internal(int fieldNumber, List list, boolean packed) + private void writeInt32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -227,7 +227,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -255,7 +255,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeFixed32List_Internal(int fieldNumber, List list, boolean packed) + private void writeFixed32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -273,7 +273,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeFixed32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeFixed32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -307,7 +307,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeUInt64List_Internal(int fieldNumber, List list, boolean packed) + private void writeUInt64List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -325,7 +325,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeUInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) + private void writeUInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -353,7 +353,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeFixed64List_Internal(int fieldNumber, List list, boolean packed) + private void writeFixed64List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -371,7 +371,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeFixed64List_Internal(int fieldNumber, LongArrayList list, boolean packed) + private void writeFixed64List_Internal(int fieldNumber, LongArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -399,7 +399,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeFloatList_Internal(int fieldNumber, List list, boolean packed) + private void writeFloatList_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -417,7 +417,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeFloatList_Internal(int fieldNumber, FloatArrayList list, boolean packed) + private void writeFloatList_Internal(int fieldNumber, FloatArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -445,7 +445,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeDoubleList_Internal(int fieldNumber, List list, boolean packed) + private void writeDoubleList_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -463,7 +463,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeDoubleList_Internal(int fieldNumber, DoubleArrayList list, boolean packed) + private void writeDoubleList_Internal(int fieldNumber, DoubleArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -497,7 +497,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeBoolList_Internal(int fieldNumber, List list, boolean packed) + private void writeBoolList_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + list.size()); @@ -515,7 +515,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeBoolList_Internal(int fieldNumber, BooleanArrayList list, boolean packed) + private void writeBoolList_Internal(int fieldNumber, BooleanArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + list.size()); @@ -572,7 +572,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeUInt32List_Internal(int fieldNumber, List list, boolean packed) + private void writeUInt32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -590,7 +590,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeUInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeUInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -630,7 +630,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeSInt32List_Internal(int fieldNumber, List list, boolean packed) + private void writeSInt32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -648,7 +648,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeSInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeSInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -759,7 +759,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeSInt64List_Internal(int fieldNumber, List list, boolean packed) + private void writeSInt64List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -777,7 +777,7 @@ abstract class BinaryWriter extends ByteOutput implements Writer { } } - private final void writeSInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) + private void writeSInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index e212ab5fe0..0c160003d3 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -137,7 +137,7 @@ public abstract class GeneratedMessageLite< // any unnecessary intermediary allocations while reducing the generated code size. /** Lazily initializes unknown fields. */ - private final void ensureUnknownFieldsInitialized() { + private void ensureUnknownFieldsInitialized() { if (unknownFields == UnknownFieldSetLite.getDefaultInstance()) { unknownFields = UnknownFieldSetLite.newInstance(); } diff --git a/kokoro/linux/bazel_distcheck/build.sh b/kokoro/linux/bazel_distcheck/build.sh deleted file mode 100755 index a50b175db2..0000000000 --- a/kokoro/linux/bazel_distcheck/build.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests using bazel-build dist archive -# -# Note that the builds use WORKSPACE to fetch external sources, not -# git submodules. - -set -eu - -use_bazel.sh 5.0.0 || true -bazel version - -# Change to repo root -cd $(dirname $0)/../../.. - -# Get kokoro scripts from repo root by default. -: ${SCRIPT_ROOT:=$(pwd)} -source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh - -# Build distribution archive -echo "============================================================" -echo -e "[[ $(date) ]] Building distribution archive...\n" -${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh build //pkg:dist_all_tar -DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz) -bazel shutdown - -# Extract the dist archive. -echo "============================================================" -echo -e "[[ $(date) ]] Extracting distribution archive...\n" - -# Construct temp directory for running the dist build. -# If you want to run locally and keep the build dir, create a directory -# and pass it in the DIST_WORK_ROOT env var. -if [[ -z ${DIST_WORK_ROOT:-} ]]; then - : ${DIST_WORK_ROOT:=$(mktemp -d)} - function dist_cleanup() { - (( $BASH_SUBSHELL == 0 )) && rm -rf ${DIST_WORK_ROOT} - } - trap dist_cleanup EXIT -fi - -DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf -mkdir -p ${DIST_WORKSPACE} -tar -C ${DIST_WORKSPACE} --strip-components=1 -axf bazel-bin/pkg/dist_all_tar.tar.gz - -echo "============================================================" -echo -e "[[ $(date) ]] Building extracted archive...\n" - -cd ${DIST_WORKSPACE} - -bazel_args=( - test - --keep_going - --test_output=errors - -- - //... - -//objectivec/... # only works on macOS - -//csharp/... # release builds require external dependencies - @com_google_protobuf_examples//... -) -${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh "${bazel_args[@]}" diff --git a/kokoro/linux/bazel_distcheck/common.cfg b/kokoro/linux/bazel_distcheck/common.cfg deleted file mode 100644 index 6b1848816d..0000000000 --- a/kokoro/linux/bazel_distcheck/common.cfg +++ /dev/null @@ -1,9 +0,0 @@ -# Common config shared by presubmit and continuous. - -bazel_setting: { - project_id: "protobuf-build" - bes_backend_address: "buildeventservice.googleapis.com" - foundry_backend_address: "remotebuildexecution.googleapis.com" - upsalite_frontend_address: "https://source.cloud.google.com" - local_execution: true -} diff --git a/kokoro/linux/bazel_distcheck/continuous.cfg b/kokoro/linux/bazel_distcheck/continuous.cfg deleted file mode 100644 index 4ea8b21006..0000000000 --- a/kokoro/linux/bazel_distcheck/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" -timeout_mins: 15 diff --git a/kokoro/linux/bazel_distcheck/presubmit.cfg b/kokoro/linux/bazel_distcheck/presubmit.cfg deleted file mode 100644 index 4ea8b21006..0000000000 --- a/kokoro/linux/bazel_distcheck/presubmit.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" -timeout_mins: 15 diff --git a/kokoro/linux/cmake/build.sh b/kokoro/linux/cmake/build.sh index 1b0ebfc5fa..523253da25 100755 --- a/kokoro/linux/cmake/build.sh +++ b/kokoro/linux/cmake/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Build file to set up and run tests based on distribution archive +# Build file to set up and run tests using CMake set -eux diff --git a/kokoro/linux/cmake_distcheck/build.sh b/kokoro/linux/cmake_distcheck/build.sh deleted file mode 100755 index 116e40b86d..0000000000 --- a/kokoro/linux/cmake_distcheck/build.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests based on distribution archive - -set -eux - -# Change to repo root -cd $(dirname $0)/../../.. - -# -# Update git submodules -# -git submodule update --init --recursive - -# -# Build distribution archive -# -# TODO: this should use Bazel-built dist archives. -date ; ./autogen.sh -date ; ./configure -date ; make dist -date - -DIST_ARCHIVE=( $(ls protobuf-*.tar.gz) ) -if (( ${#DIST_ARCHIVE[@]} != 1 )); then - echo >&2 "Distribution archive not found. ${#DIST_ARCHIVE[@]} matches:" - echo >&2 "${DIST_ARCHIVE[@]}" - exit 1 -fi - -# -# Check for all expected files -# -kokoro/common/check_missing_dist_files.sh ${DIST_ARCHIVE} - -# -# Extract to a temporary directory -# -if [[ -z ${DIST_WORK_ROOT:-} ]]; then - # If you want to preserve the extracted sources, set the DIST_WORK_ROOT - # environment variable to an existing directory that should be used. - DIST_WORK_ROOT=$(mktemp -d) - function cleanup_work_root() { - echo "Cleaning up temporary directory ${DIST_WORK_ROOT}..." - rm -rf ${DIST_WORK_ROOT} - } - trap cleanup_work_root EXIT -fi - -tar -C ${DIST_WORK_ROOT} --strip-components=1 -axf ${DIST_ARCHIVE} - -# -# Run tests using extracted sources -# -SOURCE_DIR=${DIST_WORK_ROOT} \ -CMAKE_GENERATOR=Ninja \ -CTEST_PARALLEL_LEVEL=$(nproc) \ -kokoro/common/cmake.sh - -echo "PASS" diff --git a/kokoro/linux/cmake_distcheck/continuous.cfg b/kokoro/linux/cmake_distcheck/continuous.cfg deleted file mode 100644 index 6ef4c890b4..0000000000 --- a/kokoro/linux/cmake_distcheck/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/linux/cmake_distcheck/presubmit.cfg b/kokoro/linux/cmake_distcheck/presubmit.cfg deleted file mode 100644 index 6ef4c890b4..0000000000 --- a/kokoro/linux/cmake_distcheck/presubmit.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/linux/cmake_install/build.sh b/kokoro/linux/cmake_install/build.sh index 6fdafa557e..7fdf267f74 100755 --- a/kokoro/linux/cmake_install/build.sh +++ b/kokoro/linux/cmake_install/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Build file to set up and run tests based on distribution archive +# Build file to build, install, and test using CMake. set -eux diff --git a/kokoro/linux/cmake_ninja/build.sh b/kokoro/linux/cmake_ninja/build.sh index d3a281f9ec..21cc01e56a 100755 --- a/kokoro/linux/cmake_ninja/build.sh +++ b/kokoro/linux/cmake_ninja/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Build file to set up and run tests based on distribution archive +# Build file to set up and run tests using CMake with the Ninja generator. set -eux diff --git a/kokoro/linux/cmake_shared/build.sh b/kokoro/linux/cmake_shared/build.sh new file mode 100755 index 0000000000..87dde411c7 --- /dev/null +++ b/kokoro/linux/cmake_shared/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# +# Build file to set up and run tests via CMake using shared libraries + +set -eux + +# TODO(mkruskal) Implement this. \ No newline at end of file diff --git a/kokoro/linux/cmake_shared/continuous.cfg b/kokoro/linux/cmake_shared/continuous.cfg new file mode 100644 index 0000000000..f03bd3945f --- /dev/null +++ b/kokoro/linux/cmake_shared/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_shared/presubmit.cfg b/kokoro/linux/cmake_shared/presubmit.cfg new file mode 100644 index 0000000000..f03bd3945f --- /dev/null +++ b/kokoro/linux/cmake_shared/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cpp_distcheck/build.sh b/kokoro/linux/cpp_distcheck/build.sh deleted file mode 100755 index a28843e9cb..0000000000 --- a/kokoro/linux/cpp_distcheck/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -set -ex # exit immediately on error - -# Change to repo root -cd $(dirname $0)/../../.. - -./tests.sh cpp_distcheck - -# Run tests under release docker image. -DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") -until docker pull $DOCKER_IMAGE_NAME; do sleep 10; done - -docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ - bash -l /var/local/protobuf/tests.sh cpp || FAILED="true" - -# This directory is owned by root. We need to delete it, because otherwise -# Kokoro will attempt to rsync it and fail with a permission error. -rm -rf src/core - -if [ "$FAILED" = "true" ]; then - exit 1 -fi diff --git a/kokoro/linux/cpp_distcheck/continuous.cfg b/kokoro/linux/cpp_distcheck/continuous.cfg deleted file mode 100644 index 4289f6a715..0000000000 --- a/kokoro/linux/cpp_distcheck/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/cpp_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/linux/cpp_distcheck/presubmit.cfg b/kokoro/linux/cpp_distcheck/presubmit.cfg deleted file mode 100644 index 4289f6a715..0000000000 --- a/kokoro/linux/cpp_distcheck/presubmit.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/cpp_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/linux/dist_install/build.sh b/kokoro/linux/dist_install/build.sh deleted file mode 100755 index c456ee81fa..0000000000 --- a/kokoro/linux/dist_install/build.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -set -ex # exit immediately on error - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERHUB_ORGANIZATION=protobuftesting -export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/java_stretch -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="dist_install" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/macos/cpp_distcheck/build.sh b/kokoro/macos/cpp_distcheck/build.sh deleted file mode 100755 index d729b63db1..0000000000 --- a/kokoro/macos/cpp_distcheck/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -./tests.sh cpp_distcheck diff --git a/kokoro/windows/bazel/build.bat b/kokoro/windows/bazel/build.bat new file mode 100644 index 0000000000..52b83f4666 --- /dev/null +++ b/kokoro/windows/bazel/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/linux/dist_install/continuous.cfg b/kokoro/windows/bazel/continuous.cfg similarity index 64% rename from kokoro/linux/dist_install/continuous.cfg rename to kokoro/windows/bazel/continuous.cfg index b1e0b2013d..37e89e068b 100644 --- a/kokoro/linux/dist_install/continuous.cfg +++ b/kokoro/windows/bazel/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/dist_install/build.sh" +build_file: "protobuf/kokoro/windows/cmake/build.bat" timeout_mins: 1440 diff --git a/kokoro/linux/dist_install/presubmit.cfg b/kokoro/windows/bazel/presubmit.cfg similarity index 64% rename from kokoro/linux/dist_install/presubmit.cfg rename to kokoro/windows/bazel/presubmit.cfg index b1e0b2013d..37e89e068b 100644 --- a/kokoro/linux/dist_install/presubmit.cfg +++ b/kokoro/windows/bazel/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/dist_install/build.sh" +build_file: "protobuf/kokoro/windows/cmake/build.bat" timeout_mins: 1440 diff --git a/kokoro/windows/cmake_shared/build.bat b/kokoro/windows/cmake_shared/build.bat new file mode 100644 index 0000000000..52b83f4666 --- /dev/null +++ b/kokoro/windows/cmake_shared/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/macos/cpp_distcheck/continuous.cfg b/kokoro/windows/cmake_shared/continuous.cfg similarity index 64% rename from kokoro/macos/cpp_distcheck/continuous.cfg rename to kokoro/windows/cmake_shared/continuous.cfg index 89441bcca0..37e89e068b 100644 --- a/kokoro/macos/cpp_distcheck/continuous.cfg +++ b/kokoro/windows/cmake_shared/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/cpp_distcheck/build.sh" +build_file: "protobuf/kokoro/windows/cmake/build.bat" timeout_mins: 1440 diff --git a/kokoro/macos/cpp_distcheck/presubmit.cfg b/kokoro/windows/cmake_shared/presubmit.cfg similarity index 64% rename from kokoro/macos/cpp_distcheck/presubmit.cfg rename to kokoro/windows/cmake_shared/presubmit.cfg index 89441bcca0..37e89e068b 100644 --- a/kokoro/macos/cpp_distcheck/presubmit.cfg +++ b/kokoro/windows/cmake_shared/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/cpp_distcheck/build.sh" +build_file: "protobuf/kokoro/windows/cmake/build.bat" timeout_mins: 1440 diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 2c3e06cc1b..b01d870dfa 100644 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.21.2' +__version__ = '4.21.4' diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py index 76c6802f70..0fe6a4f93c 100644 --- a/python/google/protobuf/message.py +++ b/python/google/protobuf/message.py @@ -74,7 +74,8 @@ class Message(object): __slots__ = [] - #: The :class:`google.protobuf.descriptor.Descriptor` for this message type. + #: The :class:`google.protobuf.Descriptor` + # for this message type. DESCRIPTOR = None def __deepcopy__(self, memo=None): diff --git a/src/Makefile.am b/src/Makefile.am index 4f4fc808a6..15f7aa42a3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 32:2:0 +PROTOBUF_VERSION = 32:4:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 2368446bba..0a9ee260b2 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index ac805601ae..49032949b5 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index ab524fbcef..53b524e1c3 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -56,13 +56,20 @@ namespace { PROTOBUF_CONSTINIT std::atomic g_arenaz_enabled{true}; PROTOBUF_CONSTINIT std::atomic g_arenaz_sample_parameter{1 << 10}; +PROTOBUF_CONSTINIT std::atomic + g_arenaz_config_listener{nullptr}; PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased g_exponential_biased_generator; +void TriggerThreadSafeArenazConfigListener() { + auto* listener = g_arenaz_config_listener.load(std::memory_order_acquire); + if (listener != nullptr) listener(); +} + } // namespace PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = { - .next_sample = int64_t{1} << 10, .sample_stride = int64_t{1} << 10}; + /*next_sample=*/0, /*sample_stride=*/0}; ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); } ThreadSafeArenaStats::~ThreadSafeArenaStats() = default; @@ -118,11 +125,29 @@ ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state) { return GlobalThreadSafeArenazSampler().Register(old_stride); } +void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l) { + g_arenaz_config_listener.store(l, std::memory_order_release); +} + +bool IsThreadSafeArenazEnabled() { + return g_arenaz_enabled.load(std::memory_order_acquire); +} + void SetThreadSafeArenazEnabled(bool enabled) { + SetThreadSafeArenazEnabledInternal(enabled); + TriggerThreadSafeArenazConfigListener(); +} + +void SetThreadSafeArenazEnabledInternal(bool enabled) { g_arenaz_enabled.store(enabled, std::memory_order_release); } void SetThreadSafeArenazSampleParameter(int32_t rate) { + SetThreadSafeArenazSampleParameterInternal(rate); + TriggerThreadSafeArenazConfigListener(); +} + +void SetThreadSafeArenazSampleParameterInternal(int32_t rate) { if (rate > 0) { g_arenaz_sample_parameter.store(rate, std::memory_order_release); } else { @@ -136,6 +161,11 @@ int32_t ThreadSafeArenazSampleParameter() { } void SetThreadSafeArenazMaxSamples(int32_t max) { + SetThreadSafeArenazMaxSamplesInternal(max); + TriggerThreadSafeArenazConfigListener(); +} + +void SetThreadSafeArenazMaxSamplesInternal(int32_t max) { if (max > 0) { GlobalThreadSafeArenazSampler().SetMaxSamples(max); } else { @@ -144,6 +174,10 @@ void SetThreadSafeArenazMaxSamples(int32_t max) { } } +size_t ThreadSafeArenazMaxSamples() { + return GlobalThreadSafeArenazSampler().GetMaxSamples(); +} + void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) { if (next_sample >= 0) { global_sampling_state.next_sample = next_sample; @@ -160,10 +194,16 @@ ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { return nullptr; } +void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener) {} void SetThreadSafeArenazEnabled(bool enabled) {} +void SetThreadSafeArenazEnabledInternal(bool enabled) {} +bool IsThreadSafeArenazEnabled() { return false; } void SetThreadSafeArenazSampleParameter(int32_t rate) {} +void SetThreadSafeArenazSampleParameterInternal(int32_t rate) {} int32_t ThreadSafeArenazSampleParameter() { return 0; } void SetThreadSafeArenazMaxSamples(int32_t max) {} +void SetThreadSafeArenazMaxSamplesInternal(int32_t max) {} +size_t ThreadSafeArenazMaxSamples() { return 0; } void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {} #endif // defined(PROTOBUF_ARENAZ_SAMPLE) diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index e9a4deceb0..43d1acb72f 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -199,17 +199,29 @@ inline ThreadSafeArenaStatsHandle Sample() { // Returns a global Sampler. ThreadSafeArenazSampler& GlobalThreadSafeArenazSampler(); +using ThreadSafeArenazConfigListener = void (*)(); +void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l); + // Enables or disables sampling for thread safe arenas. void SetThreadSafeArenazEnabled(bool enabled); +void SetThreadSafeArenazEnabledInternal(bool enabled); + +// Returns true if sampling is on, false otherwise. +bool IsThreadSafeArenazEnabled(); // Sets the rate at which thread safe arena will be sampled. void SetThreadSafeArenazSampleParameter(int32_t rate); +void SetThreadSafeArenazSampleParameterInternal(int32_t rate); // Returns the rate at which thread safe arena will be sampled. int32_t ThreadSafeArenazSampleParameter(); // Sets a soft max for the number of samples that will be kept. void SetThreadSafeArenazMaxSamples(int32_t max); +void SetThreadSafeArenazMaxSamplesInternal(int32_t max); + +// Returns the max number of samples that will be kept. +size_t ThreadSafeArenazMaxSamples(); // Sets the current value for when arenas should be next sampled. void SetThreadSafeArenazGlobalNextSample(int64_t next_sample); diff --git a/src/google/protobuf/arenaz_sampler_test.cc b/src/google/protobuf/arenaz_sampler_test.cc index 1b739386cd..774e70da6f 100644 --- a/src/google/protobuf/arenaz_sampler_test.cc +++ b/src/google/protobuf/arenaz_sampler_test.cc @@ -375,6 +375,7 @@ TEST(ThreadSafeArenazSamplerTest, MultiThread) { SetThreadSafeArenazEnabled(true); // Setting 1 as the parameter value means one in every two arenas would be // sampled, on average. + int32_t oldparam = ThreadSafeArenazSampleParameter(); SetThreadSafeArenazSampleParameter(1); SetThreadSafeArenazGlobalNextSample(0); auto& sampler = GlobalThreadSafeArenazSampler(); @@ -402,6 +403,95 @@ TEST(ThreadSafeArenazSamplerTest, MultiThread) { } } EXPECT_GT(count, 0); + SetThreadSafeArenazSampleParameter(oldparam); +} + +class SampleFirstArenaThread : public Thread { + protected: + void Run() override { + google::protobuf::Arena arena; + google::protobuf::ArenaSafeUniquePtr< + protobuf_test_messages::proto2::TestAllTypesProto2> + message = google::protobuf::MakeArenaSafeUnique< + protobuf_test_messages::proto2::TestAllTypesProto2>(&arena); + GOOGLE_CHECK(message != nullptr); + arena_created_.Notify(); + samples_counted_.WaitForNotification(); + } + + public: + explicit SampleFirstArenaThread(const thread::Options& options) + : Thread(options, "SampleFirstArenaThread") {} + + absl::Notification arena_created_; + absl::Notification samples_counted_; +}; + +// Test that the first arena created on a thread may and may not be chosen for +// sampling. +TEST(ThreadSafeArenazSamplerTest, SampleFirstArena) { + SetThreadSafeArenazEnabled(true); + auto& sampler = GlobalThreadSafeArenazSampler(); + + enum class SampleResult { + kSampled, + kUnsampled, + kSpoiled, + }; + + auto count_samples = [&]() { + int count = 0; + sampler.Iterate([&](const ThreadSafeArenaStats& h) { ++count; }); + return count; + }; + + auto run_sample_experiment = [&]() { + int before = count_samples(); + thread::Options options; + options.set_joinable(true); + SampleFirstArenaThread t(options); + t.Start(); + t.arena_created_.WaitForNotification(); + int during = count_samples(); + t.samples_counted_.Notify(); + t.Join(); + int after = count_samples(); + + // If we didn't get back where we were, some other thread may have + // created an arena and produced an invalid experiment run. + if (before != after) return SampleResult::kSpoiled; + + switch (during - before) { + case 1: + return SampleResult::kSampled; + case 0: + return SampleResult::kUnsampled; + default: + return SampleResult::kSpoiled; + } + }; + + constexpr int kTrials = 10000; + bool sampled = false; + bool unsampled = false; + for (int i = 0; i < kTrials; ++i) { + switch (run_sample_experiment()) { + case SampleResult::kSampled: + sampled = true; + break; + case SampleResult::kUnsampled: + unsampled = true; + break; + default: + break; + } + + // This is the success criteria for the entire test. At some point + // we sampled the first arena and at some point we did not. + if (sampled && unsampled) return; + } + EXPECT_TRUE(sampled); + EXPECT_TRUE(unsampled); } #endif // defined(PROTOBUF_ARENAZ_SAMPLE) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 51a3e652f1..50b86f7b2b 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -2205,11 +2205,12 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) { " if (IsSplitMessageDefault()) {\n" " void* chunk = " "::PROTOBUF_NAMESPACE_ID::internal::CreateSplitMessageGeneric(" - "GetArenaForAllocation(), &$1$, sizeof(Impl_::Split));\n" + "GetArenaForAllocation(), &$1$, sizeof(Impl_::Split), this, &$2$);\n" " $split$ = reinterpret_cast(chunk);\n" " }\n" "}\n", - DefaultInstanceName(descriptor_, options_, /*split=*/true)); + DefaultInstanceName(descriptor_, options_, /*split=*/true), + DefaultInstanceName(descriptor_, options_, /*split=*/false)); } GenerateVerify(printer); diff --git a/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc b/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc index e21eff17ba..56b920dbdf 100644 --- a/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc +++ b/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc @@ -74,6 +74,22 @@ TEST(DescriptorProtoHelpers, IsDescriptorOptionMessage) { EXPECT_FALSE(IsDescriptorOptionMessage(DescriptorProto::descriptor())); } +TEST(CSharpIdentifiers, UnderscoresToCamelCase) { + EXPECT_EQ("FooBar", UnderscoresToCamelCase("Foo_Bar", true)); + EXPECT_EQ("fooBar", UnderscoresToCamelCase("FooBar", false)); + EXPECT_EQ("foo123", UnderscoresToCamelCase("foo_123", false)); + // remove leading underscores + EXPECT_EQ("Foo123", UnderscoresToCamelCase("_Foo_123", true)); + // this one has slight unexpected output as it capitalises the first + // letter after consuming the underscores, but this was the existing + // behaviour so I have not changed it + EXPECT_EQ("FooBar", UnderscoresToCamelCase("___fooBar", false)); + // leave a leading underscore for identifiers that would otherwise + // be invalid because they would start with a digit + EXPECT_EQ("_123Foo", UnderscoresToCamelCase("_123_foo", true)); + EXPECT_EQ("_123Foo", UnderscoresToCamelCase("___123_foo", true)); +} + } // namespace } // namespace csharp } // namespace compiler diff --git a/src/google/protobuf/compiler/csharp/csharp_helpers.cc b/src/google/protobuf/compiler/csharp/csharp_helpers.cc index aa84dc7d82..1b58b95669 100644 --- a/src/google/protobuf/compiler/csharp/csharp_helpers.cc +++ b/src/google/protobuf/compiler/csharp/csharp_helpers.cc @@ -144,6 +144,7 @@ std::string UnderscoresToCamelCase(const std::string& input, bool cap_next_letter, bool preserve_period) { std::string result; + // Note: I distrust ctype.h due to locales. for (int i = 0; i < input.size(); i++) { if ('a' <= input[i] && input[i] <= 'z') { @@ -177,6 +178,23 @@ std::string UnderscoresToCamelCase(const std::string& input, if (input.size() > 0 && input[input.size() - 1] == '#') { result += '_'; } + + // https://github.com/protocolbuffers/protobuf/issues/8101 + // To avoid generating invalid identifiers - if the input string + // starts with _ (or multiple underscores then digit) then + // we need to preserve the underscore as an identifier cannot start + // with a digit. + // This check is being done after the loop rather than before + // to handle the case where there are multiple underscores before the + // first digit. We let them all be consumed so we can see if we would + // start with a digit. + // Note: not preserving leading underscores for all otherwise valid identifiers + // so as to not break anything that relies on the existing behaviour + if (result.size() > 0 && ('0' <= result[0] && result[0] <= '9') + && input.size() > 0 && input[0] == '_') + { + result.insert(0, 1, '_'); + } return result; } diff --git a/src/google/protobuf/compiler/csharp/csharp_map_field.cc b/src/google/protobuf/compiler/csharp/csharp_map_field.cc index a13b995da8..9efd3d524f 100644 --- a/src/google/protobuf/compiler/csharp/csharp_map_field.cc +++ b/src/google/protobuf/compiler/csharp/csharp_map_field.cc @@ -90,7 +90,7 @@ void MapFieldGenerator::GenerateMembers(io::Printer* printer) { void MapFieldGenerator::GenerateMergingCode(io::Printer* printer) { printer->Print( variables_, - "$name$_.Add(other.$name$_);\n"); + "$name$_.MergeFrom(other.$name$_);\n"); } void MapFieldGenerator::GenerateParsingCode(io::Printer* printer) { diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index b73c9de2df..51152e6525 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index f6f5da5325..439db9f000 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 87729c4893..1e4a3e18f8 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index fdc398c3d9..c5f528b9e8 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index fd75b88ace..01ecfacb1a 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 0353b74fff..70014b2abb 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -2514,6 +2514,7 @@ const Type& Reflection::GetRawNonOneof(const Message& message, } void Reflection::PrepareSplitMessageForWrite(Message* message) const { + GOOGLE_DCHECK_NE(message, schema_.default_instance_); void** split = MutableSplitField(message); const void* default_split = GetSplitField(schema_.default_instance_); if (*split == default_split) { diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index f9e07763e7..9614d015ce 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -406,7 +406,7 @@ void Tokenizer::ConsumeString(char delimiter) { case '\n': { if (!allow_multiline_strings_) { - AddError("String literals cannot cross line boundaries."); + AddError("Multiline strings are not allowed. Did you miss a \"?."); return; } NextChar(); diff --git a/src/google/protobuf/io/tokenizer_unittest.cc b/src/google/protobuf/io/tokenizer_unittest.cc index 16ba940483..6233d6ae8b 100644 --- a/src/google/protobuf/io/tokenizer_unittest.cc +++ b/src/google/protobuf/io/tokenizer_unittest.cc @@ -1067,7 +1067,8 @@ ErrorCase kErrorCases[] = { {"'\\X' foo", true, "0:2: Invalid escape sequence in string literal.\n"}, {"'\\x' foo", true, "0:3: Expected hex digits for escape sequence.\n"}, {"'foo", false, "0:4: Unexpected end of string.\n"}, - {"'bar\nfoo", true, "0:4: String literals cannot cross line boundaries.\n"}, + {"'bar\nfoo", true, + "0:4: Multiline strings are not allowed. Did you miss a \"?.\n"}, {"'\\u01' foo", true, "0:5: Expected four hex digits for \\u escape sequence.\n"}, {"'\\u01' foo", true, diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index 1279164619..5052b1c827 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -215,7 +215,9 @@ uint64_t Message::GetInvariantPerBuild(uint64_t salt) { namespace internal { void* CreateSplitMessageGeneric(Arena* arena, const void* default_split, - size_t size) { + size_t size, const void* message, + const void* default_message) { + GOOGLE_DCHECK_NE(message, default_message); void* split = (arena == nullptr) ? ::operator new(size) : arena->AllocateAligned(size); memcpy(split, default_split, size); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index b61fafb333..5a9111f6a1 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -412,7 +412,8 @@ class PROTOBUF_EXPORT Message : public MessageLite { namespace internal { // Creates and returns an allocation for a split message. void* CreateSplitMessageGeneric(Arena* arena, const void* default_split, - size_t size); + size_t size, const void* message, + const void* default_message); // Forward-declare interfaces used to implement RepeatedFieldRef. // These are protobuf internals that users shouldn't care about. diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 27a8b38bab..2674b2698e 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -521,18 +521,14 @@ void GenericTypeHandler::Merge(const std::string& from, *to = from; } -// Non-inline implementations of InternalMetadata routines -#if defined(NDEBUG) || defined(_MSC_VER) -// for opt and MSVC builds, the destructor is defined in the header. -#else +// Non-inline implementations of InternalMetadata destructor // This is moved out of the header because the GOOGLE_DCHECK produces a lot of code. -InternalMetadata::~InternalMetadata() { +void InternalMetadata::CheckedDestruct() { if (HasMessageOwnedArenaTag()) { GOOGLE_DCHECK(!HasUnknownFieldsTag()); delete reinterpret_cast(ptr_ - kMessageOwnedArenaTagMask); } } -#endif // Non-inline variants of std::string specializations for // various InternalMetadata routines. diff --git a/src/google/protobuf/metadata_lite.h b/src/google/protobuf/metadata_lite.h index af840e5065..d015dc2f9d 100644 --- a/src/google/protobuf/metadata_lite.h +++ b/src/google/protobuf/metadata_lite.h @@ -77,15 +77,19 @@ class PROTOBUF_EXPORT InternalMetadata { GOOGLE_DCHECK(!is_message_owned || arena != nullptr); } -#if defined(NDEBUG) || defined(_MSC_VER) + // To keep the ABI identical between debug and non-debug builds, + // the destructor is always defined here even though it may delegate + // to a non-inline private method. + // (see https://github.com/protocolbuffers/protobuf/issues/9947) ~InternalMetadata() { +#if defined(NDEBUG) || defined(_MSC_VER) if (HasMessageOwnedArenaTag()) { delete reinterpret_cast(ptr_ - kMessageOwnedArenaTagMask); } - } #else - ~InternalMetadata(); + CheckedDestruct(); #endif + } template void Delete() { @@ -264,6 +268,9 @@ class PROTOBUF_EXPORT InternalMetadata { PROTOBUF_NOINLINE void DoSwap(T* other) { mutable_unknown_fields()->Swap(other); } + + // Private helper with debug checks for ~InternalMetadata() + void CheckedDestruct(); }; // String Template specializations. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 94d299d3f0..7d067fd2e8 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -212,7 +212,7 @@ #ifdef PROTOBUF_VERSION #error PROTOBUF_VERSION was previously defined #endif -#define PROTOBUF_VERSION 3021002 +#define PROTOBUF_VERSION 3021004 #ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC #error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined @@ -690,15 +690,16 @@ # define PROTOBUF_CONSTEXPR constexpr # endif #else -# if defined(__cpp_constinit) +# if defined(__cpp_constinit) && !defined(__CYGWIN__) # define PROTOBUF_CONSTINIT constinit # define PROTOBUF_CONSTEXPR constexpr // Some older Clang versions incorrectly raise an error about // constant-initializing weak default instance pointers. Versions 12.0 and // higher seem to work, except that XCode 12.5.1 shows the error even though it // uses Clang 12.0.5. -# elif __has_cpp_attribute(clang::require_constant_initialization) && \ - ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \ +# elif !defined(__CYGWIN__) && \ + __has_cpp_attribute(clang::require_constant_initialization) && \ + ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \ (!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0))) # define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] # define PROTOBUF_CONSTEXPR constexpr diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index fb2e29f2ea..899cfb0122 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 7a0805d5b5..154ea45ec9 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 9d90305126..c4d26369a3 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -82,7 +82,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3021002 +#define GOOGLE_PROTOBUF_VERSION 3021004 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 3b5a469b2e..771216e61b 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index bcf1a444e0..2555602e93 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 7176f09325..de78f9a69d 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -346,6 +346,16 @@ extend TestAllExtensions { optional bytes oneof_bytes_extension = 114; } +message TestMixedFieldsAndExtensions { + optional int32 a = 1; + repeated fixed32 b = 3; + extensions 2, 4; + extend TestMixedFieldsAndExtensions { + optional int32 c = 2; + repeated fixed32 d = 4; + } +} + message TestGroup { optional group OptionalGroup = 16 { optional int32 a = 17; diff --git a/src/google/protobuf/util/json_format_proto3.proto b/src/google/protobuf/util/json_format_proto3.proto index f9c5199e3a..4df5eb9d3e 100644 --- a/src/google/protobuf/util/json_format_proto3.proto +++ b/src/google/protobuf/util/json_format_proto3.proto @@ -189,6 +189,14 @@ message TestCustomJsonName { int32 value = 1 [json_name = "@value"]; } +message TestEvilJson { + int32 regular_value = 1 [json_name = "regular_name"]; + int32 script = 2 [json_name = ""]; + int32 quotes = 3 [json_name = "unbalanced\"quotes"]; + int32 script_and_quotes = 4 + [json_name = "\""]; +} + message TestExtensions { .protobuf_unittest.TestAllExtensions extensions = 1; } diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index ca9fe731dd..99272d71ac 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -64,6 +64,11 @@ // Must be included last. #include +bool IsJson2() { + // Pay no attention to the person behind the curtain. + return false; +} + namespace google { namespace protobuf { namespace util { @@ -76,6 +81,7 @@ using ::proto3::TestOneof; using ::proto3::TestWrapper; using ::proto_util_converter::testing::MapIn; using ::testing::ElementsAre; +using ::testing::Not; using ::testing::SizeIs; // TODO(b/234474291): Use the gtest versions once that's available in OSS. @@ -274,20 +280,37 @@ TEST_P(JsonTest, TestDefaultValues) { // The ESF parser actually gets this wrong, and serializes floats whose // default value is non-finite as 0. We make sure to reproduce this bug. - EXPECT_THAT( - ToJson(protobuf_unittest::TestExtremeDefaultValues(), options), - IsOkAndHolds( - R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")" - R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)" - R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807")" - R"(,"reallySmallInt32":-2147483648,"reallySmallInt64":"-9223372036854775808",)" - R"("utf8String":"ሴ","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)" - R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)" - R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0)" - R"(,"nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0)" - R"(,"cppTrigraph":"? ? ?? ?? ??? ??/ ??-","stringWithZero":"hel\u0000lo")" - R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")" - R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})")); + if (IsJson2()) { + EXPECT_THAT( + ToJson(protobuf_unittest::TestExtremeDefaultValues(), options), + IsOkAndHolds( + R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")" + R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)" + R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807",)" + R"("utf8String":"ሴ","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)" + R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)" + R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0,)" + R"("nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0,)" + R"("cppTrigraph":"? ? ?? ?? ??? ??/ ??-","reallySmallInt32":-2147483648)" + R"(,"reallySmallInt64":"-9223372036854775808","stringWithZero":"hel\u0000lo")" + R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")" + R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})")); + } else { + EXPECT_THAT( + ToJson(protobuf_unittest::TestExtremeDefaultValues(), options), + IsOkAndHolds( + R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")" + R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)" + R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807")" + R"(,"reallySmallInt32":-2147483648,"reallySmallInt64":"-9223372036854775808",)" + R"("utf8String":"ሴ","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)" + R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)" + R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0)" + R"(,"nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0)" + R"(,"cppTrigraph":"? ? ?? ?? ??? ??/ ??-","stringWithZero":"hel\u0000lo")" + R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")" + R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})")); + } } TEST_P(JsonTest, TestPreserveProtoFieldNames) { @@ -762,6 +785,20 @@ TEST_P(JsonTest, TestParsingBrokenAny) { } )json"), StatusIs(util::StatusCode::kInvalidArgument)); + + TestAny m2; + m2.mutable_value(); + EXPECT_THAT(ToJson(m2), IsOkAndHolds(R"({"value":{}})")); + m2.mutable_value()->set_value("garbage"); + // The ESF parser does not return InvalidArgument for this error. + EXPECT_THAT(ToJson(m2), Not(StatusIs(util::StatusCode::kOk))); + + m2.Clear(); + m2.mutable_value()->set_type_url("type.googleapis.com/proto3.TestMessage"); + EXPECT_THAT( + ToJson(m2), + IsOkAndHolds( + R"({"value":{"@type":"type.googleapis.com/proto3.TestMessage"}})")); } TEST_P(JsonTest, TestFlatList) { @@ -773,7 +810,7 @@ TEST_P(JsonTest, TestFlatList) { ASSERT_OK(m); EXPECT_THAT(m->repeated_int32_value(), ElementsAre(5, 6)); - // The above flatteing behavior is supressed for google::protobuf::ListValue. + // The above flatteing behavior is suppressed for google::protobuf::ListValue. auto m2 = ToProto(R"json( { "repeatedInt32Value": [[[5]], [6]] @@ -952,6 +989,37 @@ TEST_P(JsonTest, TestParsingEnumIgnoreCase) { EXPECT_EQ(m.enum_value(), proto3::BAR); } +// This functionality is not correctly implemented by the ESF parser, so +// the test is only turned on when testing json2. +TEST_P(JsonTest, Extensions) { + if (GetParam() == Codec::kResolver || !IsJson2()) { + GTEST_SKIP(); + } + + auto m = ToProto(R"json({ + "[protobuf_unittest.TestMixedFieldsAndExtensions.c]": 42, + "a": 5, + "b": [1, 2, 3], + "[protobuf_unittest.TestMixedFieldsAndExtensions.d]": [1, 1, 2, 3, 5, 8, 13] + })json"); + ASSERT_OK(m); + EXPECT_EQ(m->a(), 5); + EXPECT_THAT(m->b(), ElementsAre(1, 2, 3)); + EXPECT_EQ(m->GetExtension(protobuf_unittest::TestMixedFieldsAndExtensions::c), + 42); + EXPECT_THAT( + m->GetRepeatedExtension(protobuf_unittest::TestMixedFieldsAndExtensions::d), + ElementsAre(1, 1, 2, 3, 5, 8, 13)); + + EXPECT_THAT( + ToJson(*m), + IsOkAndHolds( + R"({"a":5,)" + R"("[protobuf_unittest.TestMixedFieldsAndExtensions.c]":42,)" + R"("b":[1,2,3],)" + R"("[protobuf_unittest.TestMixedFieldsAndExtensions.d]":[1,1,2,3,5,8,13]})")); +} + // Parsing does NOT work like MergeFrom: existing repeated field values are // clobbered, not appended to. TEST_P(JsonTest, TestOverwriteRepeated) { @@ -1158,6 +1226,34 @@ TEST_P(JsonTest, HtmlEscape) { m.set_string_value(""); EXPECT_THAT(ToJson(m), IsOkAndHolds(R"({"stringValue":"\u003c/script\u003e"})")); + + proto3::TestEvilJson m2; + JsonPrintOptions opts; + opts.always_print_primitive_fields = true; + EXPECT_THAT( + ToJson(m2, opts), + IsOkAndHolds( + R"({"regular_name":0,"\u003c/script\u003e":0,)" + R"("unbalanced\"quotes":0,)" + R"("\"\u003cscript\u003ealert('hello!);\u003c/script\u003e":0})")); +} + +TEST_P(JsonTest, FieldOrder) { + // $ protoscope -s <<< "3: 3 22: 2 1: 1 22: 2" + std::string out; + util::Status s = BinaryToJsonString( + resolver_.get(), "type.googleapis.com/proto3.TestMessage", + "\x18\x03\xb0\x01\x02\x08\x01\xb0\x01\x02", &out); + ASSERT_OK(s); + if (IsJson2()) { + EXPECT_EQ( + out, + R"({"boolValue":true,"int64Value":"3","repeatedInt32Value":[2,2]})"); + } else { + EXPECT_EQ( + out, + R"({"int64Value":"3","repeatedInt32Value":[2],"boolValue":true,"repeatedInt32Value":[2]})"); + } } } // namespace diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 72304a56d1..f629e5e8e2 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/tests.sh b/tests.sh index ecd7e12a12..6508674031 100755 --- a/tests.sh +++ b/tests.sh @@ -45,44 +45,6 @@ build_cpp_tcmalloc() { PPROF_PATH=/usr/bin/google-pprof HEAPCHECK=strict ./protobuf-test } -build_cpp_distcheck() { - grep -q -- "-Og" src/Makefile.am && - echo "The -Og flag is incompatible with Clang versions older than 4.0." && - exit 1 - - # Initialize any submodules. - git submodule update --init --recursive - ./autogen.sh - ./configure - make dist - - # List all files that should be included in the distribution package. - git ls-files | grep "^\(java\|python\|objectivec\|csharp\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ - grep -v ".gitignore" | grep -v "java/lite/proguard.pgcfg" |\ - grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\ - grep -v "python/protobuf_distutils" | grep -v "csharp/compatibility_tests" > dist.lst - # Unzip the dist tar file. - DIST=`ls *.tar.gz` - tar -xf $DIST - cd ${DIST//.tar.gz} - # Check if every file exists in the dist tar file. - FILES_MISSING="" - for FILE in $(<../dist.lst); do - [ -f "$FILE" ] || { - echo "$FILE is not found!" - FILES_MISSING="$FILE $FILES_MISSING" - } - done - cd .. - if [ ! -z "$FILES_MISSING" ]; then - echo "Missing files in EXTRA_DIST: $FILES_MISSING" - exit 1 - fi - - # Do the regular dist-check for C++. - make distcheck -j$(nproc) -} - build_dist_install() { # Create a symlink pointing to python2 and put it at the beginning of $PATH. # This is necessary because the googletest build system involves a Python @@ -539,7 +501,6 @@ build_benchmark() { if [ "$#" -ne 1 ]; then echo " Usage: $0 { cpp | - cpp_distcheck | csharp | java_jdk7 | java_oracle7 | diff --git a/third_party/benchmark b/third_party/benchmark index 0baacde361..5b7683f49e 160000 --- a/third_party/benchmark +++ b/third_party/benchmark @@ -1 +1 @@ -Subproject commit 0baacde3618ca617da95375e0af13ce1baadea47 +Subproject commit 5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8