From 81e35132bff7c6752462f4b606af821b80ee7b98 Mon Sep 17 00:00:00 2001
From: Mike Kruskal <62662355+mkruskal-google@users.noreply.github.com>
Date: Mon, 19 Sep 2022 11:08:21 -0700
Subject: [PATCH] Improve CMake installation tests (#10615)

* Using glob to remove headers instead of cyclic file_lists

* Simplify CMake config and include missing files

* Don't remove generated proto headers

* Fix broken CMake proto dependencies instead of opting out pb.h files.

* Fixing cyclic dependency
---
 BUILD.bazel                                  |   6 +
 cmake/conformance.cmake                      |  17 +-
 cmake/tests.cmake                            |  43 +++--
 kokoro/windows/cmake/build.bat               |   4 +-
 kokoro/windows/cmake_install/build.bat       |   4 +-
 kokoro/windows/cmake_nmake/build.bat         |   4 +-
 kokoro/windows/cmake_shared/build.bat        |   4 +-
 pkg/BUILD.bazel                              |  40 +++--
 src/file_lists.cmake                         | 158 ++++---------------
 src/google/protobuf/BUILD.bazel              |  10 +-
 src/google/protobuf/compiler/cpp/BUILD.bazel |   1 +
 11 files changed, 113 insertions(+), 178 deletions(-)

diff --git a/BUILD.bazel b/BUILD.bazel
index b63b7bf984..fd9ee70aeb 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -324,6 +324,12 @@ alias(
 # Test protos
 ################################################################################
 
+alias(
+    name = "lite_test_proto_srcs",
+    actual = "//src/google/protobuf:lite_test_proto_srcs",  # proto_library
+    visibility = ["//:__subpackages__"],
+)
+
 alias(
     name = "lite_test_protos",
     actual = "//src/google/protobuf:lite_test_protos",  # proto_library
diff --git a/cmake/conformance.cmake b/cmake/conformance.cmake
index fa5c479bc6..056250ebe9 100644
--- a/cmake/conformance.cmake
+++ b/cmake/conformance.cmake
@@ -1,6 +1,8 @@
 
 add_custom_command(
-  OUTPUT ${protobuf_SOURCE_DIR}/conformance/conformance.pb.cc
+  OUTPUT
+    ${protobuf_SOURCE_DIR}/conformance/conformance.pb.h
+    ${protobuf_SOURCE_DIR}/conformance/conformance.pb.cc
   DEPENDS ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/conformance/conformance.proto
   COMMAND ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/conformance/conformance.proto
       --proto_path=${protobuf_SOURCE_DIR}/conformance
@@ -8,8 +10,11 @@ add_custom_command(
 )
 
 add_custom_command(
-  OUTPUT ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.cc
-         ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc
+  OUTPUT
+    ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.h
+    ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.cc
+    ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.h
+    ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc
   DEPENDS ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.proto
           ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.proto
   COMMAND ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.proto
@@ -21,6 +26,7 @@ add_custom_command(
 add_executable(conformance_test_runner
   ${protobuf_SOURCE_DIR}/conformance/binary_json_conformance_suite.cc
   ${protobuf_SOURCE_DIR}/conformance/binary_json_conformance_suite.h
+  ${protobuf_SOURCE_DIR}/conformance/conformance.pb.h
   ${protobuf_SOURCE_DIR}/conformance/conformance.pb.cc
   ${protobuf_SOURCE_DIR}/conformance/conformance_test.cc
   ${protobuf_SOURCE_DIR}/conformance/conformance_test_runner.cc
@@ -29,14 +35,19 @@ add_executable(conformance_test_runner
   ${protobuf_SOURCE_DIR}/conformance/text_format_conformance_suite.h
   ${protobuf_SOURCE_DIR}/conformance/third_party/jsoncpp/json.h
   ${protobuf_SOURCE_DIR}/conformance/third_party/jsoncpp/jsoncpp.cpp
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.cc
 )
 
 add_executable(conformance_cpp
+  ${protobuf_SOURCE_DIR}/conformance/conformance.pb.h
   ${protobuf_SOURCE_DIR}/conformance/conformance.pb.cc
   ${protobuf_SOURCE_DIR}/conformance/conformance_cpp.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.cc
 )
 
diff --git a/cmake/tests.cmake b/cmake/tests.cmake
index a9fdd524ca..530a66ba8b 100644
--- a/cmake/tests.cmake
+++ b/cmake/tests.cmake
@@ -43,19 +43,20 @@ endif()
 include(${protobuf_SOURCE_DIR}/src/file_lists.cmake)
 
 set(lite_test_protos
-  ${protobuf_lite_test_protos_proto_srcs}
+  ${protobuf_lite_test_protos_files}
 )
 
 set(tests_protos
-  ${protobuf_test_protos_proto_srcs}
+  ${protobuf_test_protos_files}
   ${compiler_test_protos_files}
   ${util_test_protos_files}
 )
 
 macro(compile_proto_file filename)
-  string(REPLACE .proto .pb.cc pb_file ${filename})
+  string(REPLACE .proto .pb.h pb_hdr ${filename})
+  string(REPLACE .proto .pb.cc pb_src ${filename})
   add_custom_command(
-    OUTPUT ${pb_file}
+    OUTPUT ${pb_hdr} ${pb_src}
     DEPENDS ${protobuf_PROTOC_EXE} ${filename}
     COMMAND ${protobuf_PROTOC_EXE} ${filename}
         --proto_path=${protobuf_SOURCE_DIR}/src
@@ -67,13 +68,13 @@ endmacro(compile_proto_file)
 set(lite_test_proto_files)
 foreach(proto_file ${lite_test_protos})
   compile_proto_file(${proto_file})
-  set(lite_test_proto_files ${lite_test_proto_files} ${pb_file})
+  set(lite_test_proto_files ${lite_test_proto_files} ${pb_src} ${pb_hdr})
 endforeach(proto_file)
 
 set(tests_proto_files)
 foreach(proto_file ${tests_protos})
   compile_proto_file(${proto_file})
-  set(tests_proto_files ${tests_proto_files} ${pb_file})
+  set(tests_proto_files ${tests_proto_files} ${pb_src} ${pb_hdr})
 endforeach(proto_file)
 
 add_library(protobuf-lite-test-common STATIC
@@ -85,8 +86,8 @@ target_link_libraries(protobuf-lite-test-common
 set(common_test_files
   ${test_util_hdrs}
   ${test_util_srcs}
-  ${mock_code_generator_srcs}
-  ${testing_srcs}
+  ${common_test_hdrs}
+  ${common_test_srcs}
 )
 
 add_library(protobuf-test-common STATIC
@@ -138,8 +139,8 @@ target_link_libraries(tests protobuf-lite-test-common protobuf-test-common ${pro
 
 set(test_plugin_files
   ${test_plugin_files}
-  ${mock_code_generator_srcs}
-  ${testing_srcs}
+  ${common_test_hdrs}
+  ${common_test_srcs}
 )
 
 add_executable(test_plugin ${test_plugin_files})
@@ -172,13 +173,23 @@ add_custom_target(save-installed-headers)
 add_custom_target(remove-installed-headers)
 add_custom_target(restore-installed-headers)
 
-# Explicitly skip the bootstrapping headers as it's directly used in tests
-set(_installed_hdrs ${libprotobuf_hdrs} ${libprotoc_hdrs})
-list(REMOVE_ITEM _installed_hdrs
+file(GLOB_RECURSE _local_hdrs
+  "${PROJECT_SOURCE_DIR}/src/*.h"
+  "${PROJECT_SOURCE_DIR}/src/*.inc")
+
+# Exclude the bootstrapping that are directly used by tests.
+set(_exclude_hdrs
   "${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.pb.h"
   "${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.pb.h")
 
-foreach(_hdr ${_installed_hdrs})
+# Exclude test library headers.
+list(APPEND _exclude_hdrs ${test_util_hdrs} ${lite_test_util_hdrs} ${common_test_hdrs}
+  ${compiler_test_utils_hdrs})
+foreach(_hdr ${_exclude_hdrs})
+  list(REMOVE_ITEM _local_hdrs ${_hdr})
+endforeach()
+
+foreach(_hdr ${_local_hdrs})
   string(REPLACE "${protobuf_SOURCE_DIR}/src" "" _file ${_hdr})
   set(_tmp_file "${CMAKE_BINARY_DIR}/tmp-install-test/${_file}")
   add_custom_command(TARGET remove-installed-headers PRE_BUILD
@@ -193,6 +204,6 @@ endforeach()
 
 add_dependencies(remove-installed-headers save-installed-headers)
 if(protobuf_REMOVE_INSTALLED_HEADERS)
-  add_dependencies(protobuf-lite-test-common remove-installed-headers)
-  add_dependencies(protobuf-test-common remove-installed-headers)
+  # Make sure we remove all the headers *before* any codegen occurs.
+  add_dependencies(${protobuf_PROTOC_EXE} remove-installed-headers)
 endif()
diff --git a/kokoro/windows/cmake/build.bat b/kokoro/windows/cmake/build.bat
index 49c89d3c84..d24e635d24 100644
--- a/kokoro/windows/cmake/build.bat
+++ b/kokoro/windows/cmake/build.bat
@@ -6,8 +6,8 @@ call kokoro\windows\prepare_build_win64.bat || goto :error
 @rem TODO(b/241475022) Use docker to guarantee better stability.
 @rem TODO(b/241484899) Run conformance tests in windows.
 
-md build -ea 0
-md %KOKORO_ARTIFACTS_DIR%\logs -ea 0
+md build
+md %KOKORO_ARTIFACTS_DIR%\logs
 
 cd build
 
diff --git a/kokoro/windows/cmake_install/build.bat b/kokoro/windows/cmake_install/build.bat
index 5392f9e664..c5878de338 100644
--- a/kokoro/windows/cmake_install/build.bat
+++ b/kokoro/windows/cmake_install/build.bat
@@ -6,8 +6,8 @@ call kokoro\windows\prepare_build_win64.bat || goto :error
 @rem TODO(b/241475022) Use docker to guarantee better stability.
 @rem TODO(b/241484899) Run conformance tests in windows.
 
-md build -ea 0
-md %KOKORO_ARTIFACTS_DIR%\logs -ea 0
+md build
+md %KOKORO_ARTIFACTS_DIR%\logs
 
 cd build
 
diff --git a/kokoro/windows/cmake_nmake/build.bat b/kokoro/windows/cmake_nmake/build.bat
index a0807714fc..7b03678dbc 100644
--- a/kokoro/windows/cmake_nmake/build.bat
+++ b/kokoro/windows/cmake_nmake/build.bat
@@ -6,8 +6,8 @@ call kokoro\windows\prepare_build_win64.bat || goto :error
 @rem TODO(b/241475022) Use docker to guarantee better stability.
 @rem TODO(b/241484899) Run conformance tests in windows.
 
-md build -ea 0
-md %KOKORO_ARTIFACTS_DIR%\logs -ea 0
+md build
+md %KOKORO_ARTIFACTS_DIR%\logs
 
 cd build
 
diff --git a/kokoro/windows/cmake_shared/build.bat b/kokoro/windows/cmake_shared/build.bat
index bcd5244608..d4bb2b16ee 100644
--- a/kokoro/windows/cmake_shared/build.bat
+++ b/kokoro/windows/cmake_shared/build.bat
@@ -6,8 +6,8 @@ call kokoro\windows\prepare_build_win64.bat || goto :error
 @rem TODO(b/241475022) Use docker to guarantee better stability.
 @rem TODO(b/241484899) Run conformance tests in windows.
 
-md build -ea 0
-md %KOKORO_ARTIFACTS_DIR%\logs -ea 0
+md build
+md %KOKORO_ARTIFACTS_DIR%\logs
 
 cd build
 
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
index 65dc37c0c6..4eda549da5 100644
--- a/pkg/BUILD.bazel
+++ b/pkg/BUILD.bazel
@@ -314,20 +314,18 @@ gen_file_lists(
         "//src/google/protobuf:descriptor_proto": "descriptor_proto",
         "//src/google/protobuf/compiler:plugin_proto": "plugin_proto",
         # Test libraries:
+        ":common_test": "common_test",
         ":lite_test_util": "lite_test_util",
         ":test_util": "test_util",
         # Tests and test-only protos:
         "//src/google/protobuf:full_test_srcs": "protobuf_test",
-        "//src/google/protobuf:test_protos": "protobuf_test_protos",
+        "//src/google/protobuf:test_proto_srcs": "protobuf_test_protos",
         "//src/google/protobuf:lite_test_srcs": "protobuf_lite_test",
-        "//src/google/protobuf:lite_test_protos": "protobuf_lite_test_protos",
+        "//src/google/protobuf:lite_test_proto_srcs": "protobuf_lite_test_protos",
         "//src/google/protobuf/compiler:test_srcs": "compiler_test",
-        ":compiler_annotation_test_util": "annotation_test_util",
-        ":compiler_mock_code_generator": "mock_code_generator",
         "//src/google/protobuf/compiler:test_proto_srcs": "compiler_test_protos",
         "//src/google/protobuf/compiler:test_plugin_srcs": "test_plugin",
         "//src/google/protobuf/io:test_srcs": "io_test",
-        ":testinglib": "testing",
         "//src/google/protobuf/util:test_srcs": "util_test",
         "//src/google/protobuf/util:test_proto_srcs": "util_test_protos",
         "//src/google/protobuf/stubs:test_srcs": "stubs_test",
@@ -369,6 +367,7 @@ cc_dist_library(
         "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf:arena",
         "//src/google/protobuf:protobuf_lite",
+        "//src/google/protobuf:port_def",
         "//src/google/protobuf/compiler:importer",
         "//src/google/protobuf/io",
         "//src/google/protobuf/io:gzip_stream",
@@ -384,6 +383,7 @@ cc_dist_library(
         "//src/google/protobuf/util:json_util",
         "//src/google/protobuf/util:time_util",
         "//src/google/protobuf/util:type_resolver_util",
+        "//src/google/protobuf/util/internal:constants",
         "//src/google/protobuf/util/internal:datapiece",
         "//src/google/protobuf/util/internal:default_value",
         "//src/google/protobuf/util/internal:field_mask_utility",
@@ -422,29 +422,27 @@ cc_dist_library(
     name = "test_util",
     testonly = 1,
     tags = ["manual"],
-    deps = ["//src/google/protobuf:test_util"],
-)
-
-cc_dist_library(
-    name = "compiler_annotation_test_util",
-    testonly = 1,
-    tags = ["manual"],
-    deps = ["//src/google/protobuf/compiler:annotation_test_util"],
+    deps = [
+        "//src/google/protobuf:test_util",
+        "//src/google/protobuf:test_util2",
+        "//src/google/protobuf/util/internal:expecting_objectwriter",
+        "//src/google/protobuf/util/internal:mock_error_listener",
+        "//src/google/protobuf/util/internal:type_info_test_helper",
+        "//src/google/protobuf/compiler:annotation_test_util",
+        "//src/google/protobuf/compiler/cpp:unittest_lib",
+    ],
 )
 
 cc_dist_library(
-    name = "compiler_mock_code_generator",
+    name = "common_test",
     testonly = 1,
     tags = ["manual"],
-    deps = ["//src/google/protobuf/compiler:mock_code_generator"],
+    deps = [
+        "//src/google/protobuf/testing",
+        "//src/google/protobuf/compiler:mock_code_generator",
+    ],
 )
 
-cc_dist_library(
-    name = "testinglib",
-    testonly = 1,
-    tags = ["manual"],
-    deps = ["//src/google/protobuf/testing"],
-)
 
 ################################################################################
 # Distribution sources
diff --git a/src/file_lists.cmake b/src/file_lists.cmake
index be3d61ee14..2a76933de4 100644
--- a/src/file_lists.cmake
+++ b/src/file_lists.cmake
@@ -157,6 +157,8 @@ set(libprotobuf_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/metadata_lite.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/parse_context.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/port.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/port_def.inc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/port_undef.inc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_internal.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_ops.h
@@ -179,6 +181,7 @@ set(libprotobuf_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/delimited_message_util.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/field_comparator.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/field_mask_util.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/constants.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/datapiece.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/default_value_objectwriter.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/error_listener.h
@@ -524,6 +527,20 @@ set(plugin_proto_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin_proto-descriptor-set.proto.bin
 )
 
+# //pkg:common_test
+set(common_test_srcs
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/mock_code_generator.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/file.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/googletest.cc
+)
+
+# //pkg:common_test
+set(common_test_hdrs
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/mock_code_generator.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/file.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/googletest.h
+)
+
 # //pkg:lite_test_util
 set(lite_test_util_srcs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_test_util.cc
@@ -542,12 +559,17 @@ set(lite_test_util_hdrs
 
 # //pkg:test_util
 set(test_util_srcs
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/annotation_test_util.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_tester.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/type_info_test_helper.cc
 )
 
 # //pkg:test_util
 set(test_util_hdrs
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/annotation_test_util.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/unittest.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/unittest.inc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/map_test.inc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/map_test_util.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/map_test_util.inc
@@ -555,7 +577,11 @@ set(test_util_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_tester.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util.inc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util2.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util_lite.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/expecting_objectwriter.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/mock_error_listener.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/util/internal/type_info_test_helper.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format_unittest.inc
 )
 
@@ -590,8 +616,8 @@ set(protobuf_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format_unittest.cc
 )
 
-# //src/google/protobuf:test_protos
-set(protobuf_test_protos_proto_srcs
+# //src/google/protobuf:test_proto_srcs
+set(protobuf_test_protos_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/any_test.proto
   ${protobuf_SOURCE_DIR}/src/google/protobuf/map_proto2_unittest.proto
   ${protobuf_SOURCE_DIR}/src/google/protobuf/map_unittest.proto
@@ -623,112 +649,20 @@ set(protobuf_test_protos_proto_srcs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_well_known_types.proto
 )
 
-# //src/google/protobuf:test_protos
-set(protobuf_test_protos_srcs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/any_test.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/map_proto2_unittest.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/map_unittest.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_arena.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_custom_options.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_drop_unknown_fields.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_embed_optimize_for.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_empty.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_enormous_descriptor.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_public.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lazy_dependencies.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lazy_dependencies_custom_option.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lazy_dependencies_enum.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lite_imports_nonlite.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_mset.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_mset_wire_format.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_no_field_presence.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_no_generic_services.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_optimize_for.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_preserve_unknown_enum.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_preserve_unknown_enum2.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_arena.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_arena_lite.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_lite.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_optional.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_well_known_types.proto.pb.cc
-)
-
-# //src/google/protobuf:test_protos
-set(protobuf_test_protos_hdrs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/any_test.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/map_proto2_unittest.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/map_unittest.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_arena.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_custom_options.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_drop_unknown_fields.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_embed_optimize_for.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_empty.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_enormous_descriptor.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_public.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lazy_dependencies.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lazy_dependencies_custom_option.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lazy_dependencies_enum.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lite_imports_nonlite.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_mset.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_mset_wire_format.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_no_field_presence.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_no_generic_services.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_optimize_for.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_preserve_unknown_enum.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_preserve_unknown_enum2.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_arena.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_arena_lite.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_lite.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_proto3_optional.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_well_known_types.proto.pb.h
-)
-
-# //src/google/protobuf:test_protos
-set(protobuf_test_protos_files
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/test_protos-descriptor-set.proto.bin
-)
-
 # //src/google/protobuf:lite_test_srcs
 set(protobuf_lite_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/lite_arena_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/lite_unittest.cc
 )
 
-# //src/google/protobuf:lite_test_protos
-set(protobuf_lite_test_protos_proto_srcs
+# //src/google/protobuf:lite_test_proto_srcs
+set(protobuf_lite_test_protos_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/map_lite_unittest.proto
   ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_lite.proto
   ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_public_lite.proto
   ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lite.proto
 )
 
-# //src/google/protobuf:lite_test_protos
-set(protobuf_lite_test_protos_srcs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/map_lite_unittest.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_lite.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_public_lite.proto.pb.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lite.proto.pb.cc
-)
-
-# //src/google/protobuf:lite_test_protos
-set(protobuf_lite_test_protos_hdrs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/map_lite_unittest.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_lite.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_import_public_lite.proto.pb.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_lite.proto.pb.h
-)
-
-# //src/google/protobuf:lite_test_protos
-set(protobuf_lite_test_protos_files
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/lite_test_protos-descriptor-set.proto.bin
-)
-
 # //src/google/protobuf/compiler:test_srcs
 set(compiler_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/command_line_interface_unittest.cc
@@ -750,26 +684,6 @@ set(compiler_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc
 )
 
-# //pkg:compiler_annotation_test_util
-set(annotation_test_util_srcs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/annotation_test_util.cc
-)
-
-# //pkg:compiler_annotation_test_util
-set(annotation_test_util_hdrs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/annotation_test_util.h
-)
-
-# //pkg:compiler_mock_code_generator
-set(mock_code_generator_srcs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/mock_code_generator.cc
-)
-
-# //pkg:compiler_mock_code_generator
-set(mock_code_generator_hdrs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/mock_code_generator.h
-)
-
 # //src/google/protobuf/compiler:test_proto_srcs
 set(compiler_test_protos_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto
@@ -792,18 +706,6 @@ set(io_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/io/zero_copy_stream_unittest.cc
 )
 
-# //pkg:testinglib
-set(testing_srcs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/file.cc
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/googletest.cc
-)
-
-# //pkg:testinglib
-set(testing_hdrs
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/file.h
-  ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/googletest.h
-)
-
 # //src/google/protobuf/util:test_srcs
 set(util_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/util/delimited_message_util_test.cc
diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel
index 91aaab9e30..8eaa0113e1 100644
--- a/src/google/protobuf/BUILD.bazel
+++ b/src/google/protobuf/BUILD.bazel
@@ -392,14 +392,20 @@ filegroup(
     ],
 )
 
-proto_library(
-    name = "lite_test_protos",
+filegroup(
+    name = "lite_test_proto_srcs",
     srcs = [
         "map_lite_unittest.proto",
         "unittest_import_lite.proto",
         "unittest_import_public_lite.proto",
         "unittest_lite.proto",
     ],
+    visibility = ["//:__subpackages__"],
+)
+
+proto_library(
+    name = "lite_test_protos",
+    srcs = [":lite_test_proto_srcs"],
     strip_import_prefix = "/src",
     visibility = ["//:__subpackages__"],
     deps = [
diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel
index 4a17bc10f0..8d6657d2f4 100644
--- a/src/google/protobuf/compiler/cpp/BUILD.bazel
+++ b/src/google/protobuf/compiler/cpp/BUILD.bazel
@@ -97,6 +97,7 @@ cc_library(
         "unittest.inc",
     ],
     strip_include_prefix = "/src",
+    visibility = ["//pkg:__pkg__"],
 )
 
 cc_test(