Use generated WKT code in Bazel builds (#10576)

* Use generated WKT code in Bazel builds

* Prefer src over external for genrule

* Prefer external over src for genrule

* Proper fix for windows proto path issues

BUILD.bazel

@@ -294,18 +294,8 @@ alias(
 proto_lang_toolchain(
     name = "cc_toolchain",
     blacklisted_protos = [
-        "@com_google_protobuf//:any_proto",
-        "@com_google_protobuf//:api_proto",
         "@com_google_protobuf//:compiler_plugin_proto",
         "@com_google_protobuf//:descriptor_proto",
-        "@com_google_protobuf//:duration_proto",
-        "@com_google_protobuf//:empty_proto",
-        "@com_google_protobuf//:field_mask_proto",
-        "@com_google_protobuf//:source_context_proto",
-        "@com_google_protobuf//:struct_proto",
-        "@com_google_protobuf//:timestamp_proto",
-        "@com_google_protobuf//:type_proto",
-        "@com_google_protobuf//:wrappers_proto",
     ],
     command_line = "--cpp_out=$(OUT)",
     runtime = ":protobuf",

cmake/install.cmake

@@ -47,7 +47,7 @@ include(${protobuf_SOURCE_DIR}/src/file_lists.cmake)
 set(protobuf_HEADERS
   ${libprotobuf_hdrs}
   ${libprotoc_hdrs}
-  ${wkt_protos_files}
+  ${wkt_protos_proto_srcs}
   ${descriptor_proto_proto_srcs}
   ${plugin_proto_proto_srcs}
 )

pkg/BUILD.bazel

@@ -365,7 +365,8 @@ cc_dist_library(
     }),
     tags = ["manual"],
     deps = [
-        "//src/google/protobuf",
+        "//src/google/protobuf:wkt_cc_proto",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf:arena",
         "//src/google/protobuf:protobuf_lite",
         "//src/google/protobuf/compiler:importer",

src/file_lists.cmake

@@ -11,10 +11,18 @@ endif()
 
 # //pkg:protobuf
 set(libprotobuf_srcs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/any.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/any.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/any_lite.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/api.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/duration.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/empty.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/field_mask.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/source_context.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/struct.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/timestamp.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/type.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/wrappers.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/any.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/any_lite.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arena.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_config.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arenastring.cc
@@ -24,12 +32,9 @@ set(libprotobuf_srcs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.pb.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor_database.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/duration.pb.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/dynamic_message.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/empty.pb.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/extension_set.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/extension_set_heavy.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/field_mask.pb.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/generated_enum_util.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/generated_message_bases.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/generated_message_reflection.cc
@@ -58,16 +63,12 @@ set(libprotobuf_srcs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_field.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_ptr_field.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/service.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/source_context.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/struct.pb.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/bytestream.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/common.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/stringprintf.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/structurally_valid.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/strutil.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/text_format.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/timestamp.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/type.pb.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/unknown_field_set.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/delimited_message_util.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/field_comparator.cc
@@ -91,14 +92,21 @@ set(libprotobuf_srcs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver_util.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format_lite.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/wrappers.pb.cc
 )
 
 # //pkg:protobuf
 set(libprotobuf_hdrs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/any.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/any.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/api.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/duration.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/empty.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/field_mask.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/source_context.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/struct.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/timestamp.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/type.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/wrappers.pb.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/any.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arena.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_config.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_impl.h
@@ -109,15 +117,12 @@ set(libprotobuf_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor_database.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/duration.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/dynamic_message.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/empty.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/endian.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/explicitly_constructed.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/extension_set.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/extension_set_inl.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/field_access_listener.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/field_mask.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/generated_enum_reflection.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/generated_enum_util.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/generated_message_bases.h
@@ -158,8 +163,6 @@ set(libprotobuf_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_field.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_ptr_field.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/service.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/source_context.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/struct.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/bytestream.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/callback.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/common.h
@@ -172,8 +175,6 @@ set(libprotobuf_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/stringprintf.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/strutil.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/text_format.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/timestamp.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/type.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/unknown_field_set.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/delimited_message_util.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/field_comparator.h
@@ -202,7 +203,6 @@ set(libprotobuf_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver_util.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format_lite.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/wrappers.pb.h
 )
 
 # //pkg:protobuf_lite

src/google/protobuf/BUILD.bazel

@@ -81,6 +81,57 @@ proto_library(
     strip_import_prefix = "/src",
 )
 
+# Generate code for the well-known types on demand.
+# This needs to be done in a separate genrule because we publish protoc and the
+# C++ runtime with the WKT code linked in.  We need to use a stripped down
+# compiler and runtime library to generate them, and cc_proto_library doesn't
+# support swapping out the compiler binary (even though a custom runtime can
+# be passed to proto_lang_toolchain).
+#
+# TODO(b/246826624) We still check in generated pb.h and pb.cc files purely to
+# support CMake builds.  These aren't used at all by Bazel and will be removed
+# in the future once CMake can generate them too.
+
+WELL_KNOWN_TYPES = [
+    "any",
+    "api",
+    "duration",
+    "empty",
+    "field_mask",
+    "source_context",
+    "struct",
+    "timestamp",
+    "type",
+    "wrappers",
+]
+
+genrule(
+    name = "gen_wkt_cc_sources",
+    srcs = [wkt + ".proto" for wkt in WELL_KNOWN_TYPES],
+    exec_tools = ["//src/google/protobuf/compiler:protoc_nowkt"],
+    outs =
+        [wkt + ".pb.h" for wkt in WELL_KNOWN_TYPES] +
+        [wkt + ".pb.cc" for wkt in WELL_KNOWN_TYPES],
+    cmd = """
+        $(execpath //src/google/protobuf/compiler:protoc_nowkt) \
+            --cpp_out=$(RULEDIR)/../.. \
+            --proto_path=$$(dirname $$(dirname $$(dirname $(location any.proto)))) \
+            $(SRCS)
+    """,
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "wkt_cc_proto",
+    srcs = [wkt + ".pb.cc" for wkt in WELL_KNOWN_TYPES],
+    hdrs = [wkt + ".pb.h" for wkt in WELL_KNOWN_TYPES],
+    deps = [":protobuf_nowkt"],
+    copts = COPTS,
+    include_prefix = "google/protobuf",
+    linkopts = LINK_OPTS,
+    visibility = ["//pkg:__pkg__"],
+)
+
 # Built-in runtime types
 
 proto_library(
@@ -211,19 +262,14 @@ cc_library(
 )
 
 cc_library(
-    name = "protobuf",
+    name = "protobuf_nowkt",
     srcs = [
         "any.cc",
-        "any.pb.cc",
-        "api.pb.cc",
         "descriptor.cc",
         "descriptor.pb.cc",
         "descriptor_database.cc",
-        "duration.pb.cc",
         "dynamic_message.cc",
-        "empty.pb.cc",
         "extension_set_heavy.cc",
-        "field_mask.pb.cc",
         "generated_message_bases.cc",
         "generated_message_reflection.cc",
         "generated_message_tctable_full.cc",
@@ -232,26 +278,16 @@ cc_library(
         "message.cc",
         "reflection_ops.cc",
         "service.cc",
-        "source_context.pb.cc",
-        "struct.pb.cc",
         "text_format.cc",
-        "timestamp.pb.cc",
-        "type.pb.cc",
         "unknown_field_set.cc",
         "wire_format.cc",
-        "wrappers.pb.cc",
     ],
     hdrs = [
-        "any.pb.h",
-        "api.pb.h",
         "descriptor.h",
         "descriptor.pb.h",
         "descriptor_database.h",
-        "duration.pb.h",
         "dynamic_message.h",
-        "empty.pb.h",
         "field_access_listener.h",
-        "field_mask.pb.h",
         "generated_enum_reflection.h",
         "generated_message_bases.h",
         "generated_message_reflection.h",
@@ -265,25 +301,19 @@ cc_library(
         "reflection_internal.h",
         "reflection_ops.h",
         "service.h",
-        "source_context.pb.h",
-        "struct.pb.h",
         "text_format.h",
-        "timestamp.pb.h",
-        "type.pb.h",
         "unknown_field_set.h",
         "wire_format.h",
-        "wrappers.pb.h",
     ],
     copts = COPTS,
     include_prefix = "google/protobuf",
     linkopts = LINK_OPTS,
     visibility = [
-        "//:__pkg__",
         "//pkg:__pkg__",
         "//src/google/protobuf:__subpackages__",
     ],
     deps = [
-        ":protobuf_lite",
+       ":protobuf_lite",
         "//src/google/protobuf/io",
         "//src/google/protobuf/io:gzip_stream",
         "//src/google/protobuf/io:printer",
@@ -300,6 +330,22 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "protobuf",
+    deps = [
+        ":protobuf_nowkt",
+        ":wkt_cc_proto",
+    ],
+    copts = COPTS,
+    include_prefix = "google/protobuf",
+    linkopts = LINK_OPTS,
+    visibility = [
+        "//:__pkg__",
+        "//pkg:__pkg__",
+        "//src/google/protobuf:__subpackages__",
+    ],
+)
+
 # This provides just the header files for use in projects that need to build
 # shared libraries for dynamic loading. This target is available until Bazel
 # adds native support for such use cases.

src/google/protobuf/compiler/BUILD.bazel

@@ -11,7 +11,7 @@ load(
 )
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test")
-load("//build_defs:cpp_opts.bzl", "COPTS")
+load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS")
 
 proto_library(
     name = "plugin_proto",
@@ -37,7 +37,7 @@ cc_library(
     include_prefix = "google/protobuf/compiler",
     visibility = ["//visibility:public"],
     deps = [
-        "//src/google/protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -59,7 +59,7 @@ cc_library(
     include_prefix = "google/protobuf/compiler",
     visibility = ["//visibility:public"],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -82,26 +82,22 @@ cc_library(
     deps = [
         ":code_generator",
         ":importer",
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "@com_google_absl//absl/strings",
     ],
 )
 
 cc_library(
-    name = "protoc_lib",
+    name = "protoc_lib_nowkt",
     srcs = [
         "main.cc",
     ],
     copts = COPTS,
-    visibility = [
-        "//:__pkg__",
-        "//pkg:__pkg__",
-    ],
     deps = [
         ":code_generator",
         ":command_line_interface",
         ":importer",
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler/cpp",
         "//src/google/protobuf/compiler/csharp",
         "//src/google/protobuf/compiler/java",
@@ -112,6 +108,29 @@ cc_library(
     ],
 )
 
+cc_binary(
+    name = "protoc_nowkt",
+    copts = COPTS,
+    linkopts = LINK_OPTS + PROTOC_LINK_OPTS,
+    visibility = [
+        "//src/google/protobuf:__pkg__",
+    ],
+    deps = [":protoc_lib_nowkt"],
+)
+
+cc_library(
+    name = "protoc_lib",
+    copts = COPTS,
+    visibility = [
+        "//:__pkg__",
+        "//pkg:__pkg__",
+    ],
+    deps = [
+        "//:protobuf",
+        ":protoc_lib_nowkt",
+    ],
+)
+
 # Note: this is an alias for now. In the future, this rule will become the
 # cc_binary for protoc, and //:protoc will become an alias.
 alias(

src/google/protobuf/compiler/cpp/BUILD.bazel

@@ -53,7 +53,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/container:flat_hash_map",

src/google/protobuf/compiler/csharp/BUILD.bazel

@@ -56,7 +56,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",

src/google/protobuf/compiler/java/BUILD.bazel

@@ -81,7 +81,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",

src/google/protobuf/compiler/objectivec/BUILD.bazel

@@ -44,7 +44,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/strings",
     ],

src/google/protobuf/compiler/php/BUILD.bazel

@@ -17,7 +17,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/strings",
     ],

src/google/protobuf/compiler/python/BUILD.bazel

@@ -25,7 +25,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/synchronization",

src/google/protobuf/compiler/ruby/BUILD.bazel

@@ -17,7 +17,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
-        "//:protobuf",
+        "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
     ],
 )