From b8edc7153f017c4cfb1be3223e51e58a22f62bdd Mon Sep 17 00:00:00 2001 From: Protobuf Team Date: Wed, 16 Mar 2022 08:24:52 -0700 Subject: [PATCH] Internal change PiperOrigin-RevId: 435056074 --- BUILD | 12 +- bazel/build_defs.bzl | 60 +++---- bazel/py_proto_library.bzl | 5 +- bazel/pyproto_test_wrapper.bzl | 8 +- bazel/upb_proto_library.bzl | 15 +- benchmarks/BUILD | 154 ++++++++++-------- benchmarks/build_defs.bzl | 23 +-- python/BUILD | 9 +- python/pb_unit_tests/BUILD | 4 +- .../descriptor_pool_test_wrapper.py | 15 +- .../pb_unit_tests/text_format_test_wrapper.py | 4 +- python/protobuf.h | 9 +- upb/util/BUILD | 1 - upb/util/def_to_proto_test.proto | 1 - upbc/BUILD | 6 +- 15 files changed, 176 insertions(+), 150 deletions(-) diff --git a/BUILD b/BUILD index 0a7803f169..4cabc12119 100644 --- a/BUILD +++ b/BUILD @@ -28,7 +28,6 @@ load( "UPB_DEFAULT_COPTS", "UPB_DEFAULT_CPPOPTS", "make_shell_script", - "upb_amalgamation", # copybara:strip_for_google3 ) load( "//bazel:upb_proto_library.bzl", @@ -38,6 +37,13 @@ load( "upb_proto_reflection_library", ) +# begin:github_only +load( + "//bazel:build_defs.bzl", + "upb_amalgamation", +) +# end:github_only + licenses(["notice"]) exports_files([ @@ -535,7 +541,7 @@ cc_library( # Amalgamation ################################################################# -# copybara:strip_for_google3_begin +# begin:github_only upb_amalgamation( name = "gen_amalgamation", @@ -640,4 +646,4 @@ filegroup( visibility = ["//cmake:__pkg__"], ) -# copybara:strip_end +# end:github_only diff --git a/bazel/build_defs.bzl b/bazel/build_defs.bzl index 5b35b277ac..b5bc64f071 100644 --- a/bazel/build_defs.bzl +++ b/bazel/build_defs.bzl @@ -27,39 +27,40 @@ load(":upb_proto_library.bzl", "GeneratedSrcsInfo") +_DEFAULT_CPPOPTS = [] +_DEFAULT_COPTS = [] + +# begin:github_only +_DEFAULT_CPPOPTS.extend([ + "-Wextra", + # "-Wshorten-64-to-32", # not in GCC (and my Kokoro images doesn't have Clang) + "-Werror", + "-Wno-long-long", +]) +_DEFAULT_COPTS.extend([ + "-std=c99", + "-pedantic", + "-Werror=pedantic", + "-Wall", + "-Wstrict-prototypes", + # GCC (at least) emits spurious warnings for this that cannot be fixed + # without introducing redundant initialization (with runtime cost): + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80635 + #"-Wno-maybe-uninitialized", +]) +# end:github_only + UPB_DEFAULT_CPPOPTS = select({ "//:windows": [], - "//conditions:default": [ - # copybara:strip_for_google3_begin - "-Wextra", - # "-Wshorten-64-to-32", # not in GCC (and my Kokoro images doesn't have Clang) - "-Werror", - "-Wno-long-long", - # copybara:strip_end - ], + "//conditions:default": _DEFAULT_CPPOPTS, }) UPB_DEFAULT_COPTS = select({ "//:windows": [], "//:fasttable_enabled_setting": ["-std=gnu99", "-DUPB_ENABLE_FASTTABLE"], - "//conditions:default": [ - # copybara:strip_for_google3_begin - "-std=c99", - "-pedantic", - "-Werror=pedantic", - "-Wall", - "-Wstrict-prototypes", - # GCC (at least) emits spurious warnings for this that cannot be fixed - # without introducing redundant initialization (with runtime cost): - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80635 - #"-Wno-maybe-uninitialized", - # copybara:strip_end - ], + "//conditions:default": _DEFAULT_COPTS, }) -def _librule(name): - return name + "_lib" - runfiles_init = """\ # --- begin runfiles.bash initialization v2 --- # Copy-pasted from the Bazel Bash runfiles library v2. @@ -94,18 +95,7 @@ def _get_real_roots(files): roots[real_root] = True return roots.keys() -def _remove_prefix(str, prefix): - if not str.startswith(prefix): - fail("%s doesn't start with %s" % (str, prefix)) - return str[len(prefix):] - -def _remove_suffix(str, suffix): - if not str.endswith(suffix): - fail("%s doesn't end with %s" % (str, suffix)) - return str[:-len(suffix)] - def make_shell_script(name, contents, out): - contents = runfiles_init + contents # copybara:strip_for_google3 contents = contents.replace("$", "$$") native.genrule( name = "gen_" + name, diff --git a/bazel/py_proto_library.bzl b/bazel/py_proto_library.bzl index 8ef4d9708d..72351256cd 100644 --- a/bazel/py_proto_library.bzl +++ b/bazel/py_proto_library.bzl @@ -34,7 +34,10 @@ business of vending py_proto_library(), so we keep it private to upb. """ load("@bazel_skylib//lib:paths.bzl", "paths") -load("@rules_proto//proto:defs.bzl", "ProtoInfo") # copybara:strip_for_google3 + +# begin:github_only +load("@rules_proto//proto:defs.bzl", "ProtoInfo") +# end:github_only # Generic support code ######################################################### diff --git a/bazel/pyproto_test_wrapper.bzl b/bazel/pyproto_test_wrapper.bzl index e0d01ca225..629ee9b2c6 100644 --- a/bazel/pyproto_test_wrapper.bzl +++ b/bazel/pyproto_test_wrapper.bzl @@ -1,4 +1,4 @@ -# copybara:strip_for_google3_begin +# begin:github_only def pyproto_test_wrapper(name): src = name + "_wrapper.py" @@ -16,7 +16,9 @@ def pyproto_test_wrapper(name): ], ) -# copybara:replace_for_google3_begin +# end:github_only + +# begin:google_only # # def pyproto_test_wrapper(name): # src = name + "_wrapper.py" @@ -30,4 +32,4 @@ def pyproto_test_wrapper(name): # ], # ) # -# copybara:replace_for_google3_end +# end:google_only diff --git a/bazel/upb_proto_library.bzl b/bazel/upb_proto_library.bzl index 0a30eee611..82af0e6af4 100644 --- a/bazel/upb_proto_library.bzl +++ b/bazel/upb_proto_library.bzl @@ -30,11 +30,16 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") -load("@rules_proto//proto:defs.bzl", "ProtoInfo") # copybara:strip_for_google3 # Generic support code ######################################################### -_is_bazel = True # copybara:replace_for_google3 _is_bazel = False +# begin:github_only +_is_google3 = False +# end:github_only + +# begin:google_only +# _is_google3 = True +# end:google_only def _get_real_short_path(file): # For some reason, files from other archives have short paths that look like: @@ -97,7 +102,7 @@ def _cc_library_func(ctx, name, hdrs, srcs, copts, dep_ccinfos): blaze_only_args = {} - if not _is_bazel: + if _is_google3: blaze_only_args["grep_includes"] = ctx.file._grep_includes (compilation_context, compilation_outputs) = cc_common.compile( @@ -269,11 +274,11 @@ def _upb_proto_reflection_library_aspect_impl(target, ctx): return _upb_proto_aspect_impl(target, ctx, "upbdefs", _UpbDefsWrappedCcInfo, _WrappedDefsGeneratedSrcsInfo) def _maybe_add(d): - if not _is_bazel: + if _is_google3: d["_grep_includes"] = attr.label( allow_single_file = True, cfg = "exec", - default = "//tools/cpp:grep-includes", + default = "@bazel_tools//tools/cpp:grep-includes", ) return d diff --git a/benchmarks/BUILD b/benchmarks/BUILD index ae4b1995b2..a7852c776d 100644 --- a/benchmarks/BUILD +++ b/benchmarks/BUILD @@ -23,6 +23,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# begin:google_only +# load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library") +# end:google_only + load( "//bazel:upb_proto_library.bzl", "upb_proto_library", @@ -30,10 +34,10 @@ load( ) load( ":build_defs.bzl", - "tmpl_cc_binary", "cc_optimizefor_proto_library", "expand_suffixes", "proto_library", + "tmpl_cc_binary", ) licenses(["notice"]) @@ -87,6 +91,7 @@ cc_test( "//:reflection", "@com_github_google_benchmark//:benchmark_main", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_protobuf//:protobuf", ], ) @@ -121,7 +126,6 @@ py_binary( genrule( name = "do_gen_synthetic_protos", - tools = [":gen_synthetic_protos"], outs = [ "100_msgs.proto", "200_msgs.proto", @@ -129,6 +133,7 @@ genrule( "200_fields.proto", ], cmd = "$(execpath :gen_synthetic_protos) $(RULEDIR)", + tools = [":gen_synthetic_protos"], ) proto_library( @@ -157,82 +162,87 @@ proto_library( ) [( -upb_proto_library( - name = k + "_upb_proto", - deps = [":" + k + "_proto"], -), -cc_proto_library( - name = k + "_cc_proto", - deps = [":" + k + "_proto"], -), -tmpl_cc_binary( - name = k + "_upb_binary", - testonly = 1, - gen = ":gen_upb_binary_c", - args = [ - package_name() + "/" + k + ".upb.h", - "upb_benchmark_" + v, - ], - deps = [ - ":" + k + "_upb_proto", - ], -), -tmpl_cc_binary( - name = k + "_protobuf_binary", - testonly = 1, - gen = ":gen_protobuf_binary_cc", - args = [ - package_name() + "/" + k + ".pb.h", - "upb_benchmark::" + v, - ], - deps = [ - ":" + k + "_cc_proto", - ], -), -cc_optimizefor_proto_library( - srcs = [k + ".proto"], - outs = [k + "_lite.proto"], - name = k + "_cc_lite_proto", - optimize_for = "LITE_RUNTIME", -), -tmpl_cc_binary( - name = k + "_lite_protobuf_binary", - testonly = 1, - gen = ":gen_protobuf_binary_cc", - args = [ - package_name() + "/" + k + "_lite.pb.h", - "upb_benchmark::" + v, - ], - deps = [ - ":" + k + "_cc_lite_proto", - ], -), -cc_optimizefor_proto_library( - srcs = [k + ".proto"], - outs = [k + "_codesize.proto"], - name = k + "_cc_codesize_proto", - optimize_for = "CODE_SIZE", -), -tmpl_cc_binary( - name = k + "_codesize_protobuf_binary", - testonly = 1, - gen = ":gen_protobuf_binary_cc", - args = [ - package_name() + "/" + k + "_codesize.pb.h", - "upb_benchmark::" + v, - ], - deps = [ - ":" + k + "_cc_codesize_proto", - ], -) + upb_proto_library( + name = k + "_upb_proto", + deps = [":" + k + "_proto"], + ), + cc_proto_library( + name = k + "_cc_proto", + deps = [":" + k + "_proto"], + ), + tmpl_cc_binary( + name = k + "_upb_binary", + testonly = 1, + args = [ + package_name() + "/" + k + ".upb.h", + "upb_benchmark_" + v, + ], + gen = ":gen_upb_binary_c", + deps = [ + ":" + k + "_upb_proto", + ], + ), + tmpl_cc_binary( + name = k + "_protobuf_binary", + testonly = 1, + args = [ + package_name() + "/" + k + ".pb.h", + "upb_benchmark::" + v, + ], + gen = ":gen_protobuf_binary_cc", + deps = [ + ":" + k + "_cc_proto", + ], + ), + cc_optimizefor_proto_library( + name = k + "_cc_lite_proto", + srcs = [k + ".proto"], + outs = [k + "_lite.proto"], + optimize_for = "LITE_RUNTIME", + ), + tmpl_cc_binary( + name = k + "_lite_protobuf_binary", + testonly = 1, + args = [ + package_name() + "/" + k + "_lite.pb.h", + "upb_benchmark::" + v, + ], + gen = ":gen_protobuf_binary_cc", + deps = [ + ":" + k + "_cc_lite_proto", + ], + ), + cc_optimizefor_proto_library( + name = k + "_cc_codesize_proto", + srcs = [k + ".proto"], + outs = [k + "_codesize.proto"], + optimize_for = "CODE_SIZE", + ), + tmpl_cc_binary( + name = k + "_codesize_protobuf_binary", + testonly = 1, + args = [ + package_name() + "/" + k + "_codesize.pb.h", + "upb_benchmark::" + v, + ], + gen = ":gen_protobuf_binary_cc", + deps = [ + ":" + k + "_cc_codesize_proto", + ], + ), ) for k, v in SIZE_BENCHMARKS.items()] genrule( - testonly = 1, name = "size_data", + testonly = 1, srcs = expand_suffixes( SIZE_BENCHMARKS.keys(), - suffixes = ["_upb_binary", "_protobuf_binary", "_lite_protobuf_binary", "_codesize_protobuf_binary"], + suffixes = [ + "_upb_binary", + "_protobuf_binary", + "_lite_protobuf_binary", + "_codesize_protobuf_binary", + ], ), outs = ["size_data.txt"], # We want --format=GNU which counts rodata with data, not text. diff --git a/benchmarks/build_defs.bzl b/benchmarks/build_defs.bzl index 93da39f086..3fc4dd22a8 100644 --- a/benchmarks/build_defs.bzl +++ b/benchmarks/build_defs.bzl @@ -23,19 +23,21 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# copybara:insert_for_google3_begin +# begin:google_only # load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", _cc_proto_library = "cc_proto_library") -# copybara:insert_end +# +# _is_google3 = True +# end:google_only -# copybara:strip_for_google3_begin +# begin:github_only _cc_proto_library = native.cc_proto_library -# copybara:strip_end +_is_google3 = False +# end:github_only def proto_library(**kwargs): + if _is_google3: + kwargs["cc_api_version"] = 2 native.proto_library( - # copybara:insert_for_google3_begin - # cc_api_version = 2, - # copybara:insert_end **kwargs ) @@ -48,11 +50,10 @@ def tmpl_cc_binary(name, gen, args, replacements = [], **kwargs): cmd = "$(location " + gen + ") " + " ".join(args) + " > $@", ) + if _is_google3: + kwargs["malloc"] = "//base:system_malloc" + kwargs["features"] = ["-static_linking_mode"] native.cc_binary( - # copybara:insert_for_google3_begin - # malloc="//base:system_malloc", - # features = ["-static_linking_mode"], - # copybara:insert_end name = name, srcs = srcs, **kwargs diff --git a/python/BUILD b/python/BUILD index cb48bc6109..ac971132b5 100644 --- a/python/BUILD +++ b/python/BUILD @@ -23,9 +23,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# begin:github_only load("//bazel:py_proto_library.bzl", "py_proto_library") -load("//bazel:py_extension.bzl", "py_extension") # copybara:strip_for_google3 +load("//bazel:py_extension.bzl", "py_extension") load("@rules_python//python:packaging.bzl", "py_wheel") +# end:github_only licenses(["notice"]) @@ -69,7 +71,7 @@ py_extension( srcs = ["api_implementation.c"], ) -# copybara:strip_for_google3_begin +# begin:github_only py_test( name = "minimal_test", @@ -120,7 +122,6 @@ py_library( visibility = ["//python:__subpackages__"], ) - py_proto_library( name = "well_known_proto_pb2", deps = [ @@ -160,4 +161,4 @@ py_wheel( ], ) -# copybara:strip_end +# end:github_only diff --git a/python/pb_unit_tests/BUILD b/python/pb_unit_tests/BUILD index ecb1d7b900..1a788fe4e5 100644 --- a/python/pb_unit_tests/BUILD +++ b/python/pb_unit_tests/BUILD @@ -33,7 +33,9 @@ pyproto_test_wrapper(name = "descriptor_pool_test") pyproto_test_wrapper(name = "descriptor_test") -pyproto_test_wrapper(name = "generator_test") # copybara:strip_for_google3 +# begin:github_only +pyproto_test_wrapper(name = "generator_test") +# end:github_only pyproto_test_wrapper(name = "json_format_test") diff --git a/python/pb_unit_tests/descriptor_pool_test_wrapper.py b/python/pb_unit_tests/descriptor_pool_test_wrapper.py index 800ff4826c..69b573c696 100644 --- a/python/pb_unit_tests/descriptor_pool_test_wrapper.py +++ b/python/pb_unit_tests/descriptor_pool_test_wrapper.py @@ -27,9 +27,9 @@ from google.protobuf.internal import descriptor_pool_test import unittest import copy -# copybara:insert_for_google3_begin +# begin:google_only # from google3.testing.pybase import googletest -# copybara:insert_end +# end:google_only # This is testing that certain methods unconditionally throw TypeError. # In the new extension we simply don't define them at all. @@ -37,9 +37,12 @@ descriptor_pool_test.AddDescriptorTest.testAddTypeError.__unittest_expecting_fai descriptor_pool_test.SecondaryDescriptorFromDescriptorDB.testErrorCollector.__unittest_expecting_failure__ = True +# begin:github_only if __name__ == '__main__': - # copybara:strip_for_google3_begin unittest.main(module=descriptor_pool_test, verbosity=2) - # copybara:replace_for_google3_begin - # googletest.main() - # copybara:replace_for_google3_end +# end:github_only + +# begin:google_only +# if __name__ == '__main__': +# googletest.main() +# end:google_only diff --git a/python/pb_unit_tests/text_format_test_wrapper.py b/python/pb_unit_tests/text_format_test_wrapper.py index 1183a5b8be..e108e09801 100644 --- a/python/pb_unit_tests/text_format_test_wrapper.py +++ b/python/pb_unit_tests/text_format_test_wrapper.py @@ -29,12 +29,12 @@ import unittest # These rely on the UnknownFields accessor, which we are trying to deprecate. text_format_test.OnlyWorksWithProto2RightNowTests.testPrintUnknownFields.__unittest_expecting_failure__ = True -# copybara:strip_for_google3_begin +# begin:github_only from google.protobuf.internal import _parameterized # copybara:strip_for_google3 sep = _parameterized._SEPARATOR getattr(text_format_test.TextFormatMessageToStringTests, "testPrintUnknownFieldsEmbeddedMessageInBytes" + sep + "0").__unittest_expecting_failure__ = True getattr(text_format_test.TextFormatMessageToStringTests, "testPrintUnknownFieldsEmbeddedMessageInBytes" + sep + "1").__unittest_expecting_failure__ = True -# copybara:strip_end +# end:github_only if __name__ == '__main__': unittest.main(module=text_format_test, verbosity=2) diff --git a/python/protobuf.h b/python/protobuf.h index d09ab0e8db..0b87646895 100644 --- a/python/protobuf.h +++ b/python/protobuf.h @@ -34,19 +34,22 @@ #include "python/python.h" #include "upb/table_internal.h" -// copybara:strip_for_google3_begin + +// begin:github_only #define PYUPB_PROTOBUF_PUBLIC_PACKAGE "google.protobuf" #define PYUPB_PROTOBUF_INTERNAL_PACKAGE "google.protobuf.internal" #define PYUPB_DESCRIPTOR_PROTO_PACKAGE "google.protobuf" #define PYUPB_DESCRIPTOR_MODULE "google.protobuf.descriptor_pb2" #define PYUPB_MODULE_NAME "google.protobuf.pyext._message" -// copybara:replace_for_google3_begin +// end:github_only + +// begin:google_only // #define PYUPB_PROTOBUF_PUBLIC_PACKAGE "google3.net.proto2.python.public" // #define PYUPB_PROTOBUF_INTERNAL_PACKAGE "google3.net.proto2.python.internal" // #define PYUPB_DESCRIPTOR_PROTO_PACKAGE "proto2" // #define PYUPB_DESCRIPTOR_MODULE "google3.net.proto2.proto.descriptor_pb2" // #define PYUPB_MODULE_NAME "google3.third_party.upb.python._message" -// copybara:replace_for_google3_end +// end:google_only #define PYUPB_RETURN_OOM return PyErr_SetNone(PyExc_MemoryError), NULL diff --git a/upb/util/BUILD b/upb/util/BUILD index e39044a111..9c34822932 100644 --- a/upb/util/BUILD +++ b/upb/util/BUILD @@ -25,7 +25,6 @@ proto_library( "def_to_proto_public_import_test.proto", "def_to_proto_regular_import_test.proto", "def_to_proto_test.proto", - "def_to_proto_weak_import_test.proto", # copybara:strip_for_google3 ], ) diff --git a/upb/util/def_to_proto_test.proto b/upb/util/def_to_proto_test.proto index 56ebbd9522..d0bcb3b8b5 100644 --- a/upb/util/def_to_proto_test.proto +++ b/upb/util/def_to_proto_test.proto @@ -29,7 +29,6 @@ syntax = "proto2"; import "upb/util/def_to_proto_regular_import_test.proto"; import public "upb/util/def_to_proto_public_import_test.proto"; -import weak "upb/util/def_to_proto_weak_import_test.proto"; // copybara:strip_for_google3 package pkg; diff --git a/upbc/BUILD b/upbc/BUILD index 23b26214b9..0b98f2902e 100644 --- a/upbc/BUILD +++ b/upbc/BUILD @@ -32,12 +32,12 @@ licenses(["notice"]) cc_library( name = "common", - hdrs = ["common.h"], srcs = ["common.cc"], + hdrs = ["common.h"], copts = UPB_DEFAULT_CPPOPTS, deps = [ - "@com_google_protobuf//:protobuf", "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", ], ) @@ -49,6 +49,8 @@ cc_binary( deps = [ ":common", "//:mini_table", + "//:port", + "//:upb", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set",