Add a test for bazel-built dist archives (#9929)

This also fixes some packaging rules that are needed for the result to be buildable.

BUILD.bazel

@@ -1284,6 +1284,7 @@ pkg_files(
         "generate_descriptor_proto.sh",
         "maven_install.json",
         "update_file_lists.sh",
+        "//third_party:BUILD.bazel",
         "//third_party:zlib.BUILD",
         "//util/python:BUILD.bazel",
     ],

benchmarks/BUILD.bazel

@@ -75,7 +75,6 @@ pkg_files(
         ],
         exclude = [
             "__init__.py",  # not in autotools dist
-            "BUILD.bazel",
             "go/*",
         ],
     ),

benchmarks/cpp/BUILD.bazel

@@ -17,7 +17,10 @@ cc_binary(
 
 pkg_files(
     name = "dist_files",
-    srcs = ["cpp_benchmark.cc"],
+    srcs = [
+        "BUILD.bazel",
+        "cpp_benchmark.cc",
+    ],
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//benchmarks:__pkg__"],
 )

benchmarks/datasets/google_message1/proto2/BUILD.bazel

@@ -46,7 +46,7 @@ java_proto_library(
 
 pkg_files(
     name = "dist_files",
-    srcs = glob(["*.proto"]),
+    srcs = glob(["*"]),
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//benchmarks:__pkg__"],
 )

benchmarks/datasets/google_message1/proto3/BUILD.bazel

@@ -46,7 +46,7 @@ java_proto_library(
 
 pkg_files(
     name = "dist_files",
-    srcs = glob(["*.proto"]),
+    srcs = glob(["*"]),
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//benchmarks:__pkg__"],
 )

benchmarks/datasets/google_message2/BUILD.bazel

@@ -46,7 +46,7 @@ java_proto_library(
 
 pkg_files(
     name = "dist_files",
-    srcs = glob(["*.proto"]),
+    srcs = glob(["*"]),
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//benchmarks:__pkg__"],
 )

benchmarks/datasets/google_message3/BUILD.bazel

@@ -52,7 +52,7 @@ java_proto_library(
 
 pkg_files(
     name = "dist_files",
-    srcs = glob(["*.proto"]),
+    srcs = glob(["*"]),
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//benchmarks:__pkg__"],
 )

benchmarks/datasets/google_message4/BUILD.bazel

@@ -47,7 +47,7 @@ java_proto_library(
 
 pkg_files(
     name = "dist_files",
-    srcs = glob(["*.proto"]),
+    srcs = glob(["*"]),
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//benchmarks:__pkg__"],
 )

conformance/BUILD.bazel

@@ -1,7 +1,13 @@
 # Conformance testing for Protobuf.
 
-load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library", "cc_proto_library")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library")
-load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+load(
+    "@rules_pkg//:mappings.bzl",
+    "pkg_attributes",
+    "pkg_filegroup",
+    "pkg_files",
+    "strip_prefix",
+)
 
 exports_files([
     "conformance_test_runner.sh",
@@ -145,16 +151,33 @@ pkg_files(
     srcs = glob(
         ["**/*"],
         exclude = [
+            # Handled by dist_scripts:
+            "conformance_test_runner.sh",
+
             # The following are not in autotools dist:
-            "conformance/autoload.php",
+            "autoload.php",
-            "conformance/conformance_nodejs.js",
+            "conformance_nodejs.js",
-            "conformance/conformance_test_runner.sh",
+            "failure_list_jruby.txt",
-            "conformance/failure_list_java_lite.txt",
+            "update_failure_list.py",
-            "conformance/failure_list_jruby.txt",
-            "conformance/text_format_failure_list_*.txt",
-            "conformance/update_failure_list.py",
         ],
     ),
     strip_prefix = strip_prefix.from_root(""),
     visibility = ["//pkg:__pkg__"],
 )
+
+pkg_files(
+    name = "dist_scripts",
+    srcs = ["conformance_test_runner.sh"],
+    attributes = pkg_attributes(mode = "0555"),
+    strip_prefix = strip_prefix.from_root(""),
+    visibility = ["//pkg:__pkg__"],
+)
+
+pkg_filegroup(
+    name = "all_dist_files",
+    srcs = [
+        ":dist_files",
+        ":dist_scripts",
+    ],
+    visibility = ["//pkg:__pkg__"],
+)

kokoro/linux/bazel_distcheck/build.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Build file to set up and run tests using bazel-build dist archive
+#
+# Note that the builds use WORKSPACE to fetch external sources, not
+# git submodules.
+
+set -eux
+
+BUILD_ONLY_TARGETS=(
+  //pkg:all
+  //:protoc
+  //:protobuf
+  //:protobuf_python
+)
+
+TEST_TARGETS=(
+  //build_defs:all
+  //conformance:all
+  //java:tests
+  //:protobuf_test
+)
+
+use_bazel.sh 5.0.0 || true
+bazel version
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Construct temp directory for running the dist build.
+# If you want to run locally and keep the build dir, create a directory
+# and pass it in the DIST_WORK_ROOT env var.
+if [[ -z ${DIST_WORK_ROOT:-} ]]; then
+  : ${DIST_WORK_ROOT:=$(mktemp -d)}
+  function dist_cleanup() {
+    rm -rf ${DIST_WORK_ROOT}
+  }
+  trap dist_cleanup EXIT
+fi
+
+# Let Bazel share the distdir.
+TMP_DISTDIR=${DIST_WORK_ROOT}/bazel-distdir
+mkdir -p ${TMP_DISTDIR}
+
+# Build distribution archive
+date
+bazel fetch --distdir=${TMP_DISTDIR} //pkg:dist_all_tar
+bazel build --distdir=${TMP_DISTDIR} //pkg:dist_all_tar
+DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz)
+bazel shutdown
+
+# The `pkg_tar` rule emits a symlink based on the rule name. The actual
+# file is named with the current version.
+date
+echo "Resolved archive path: ${DIST_ARCHIVE}"
+
+# Extract the dist archive.
+date
+DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf
+mkdir -p ${DIST_WORKSPACE}
+tar -C ${DIST_WORKSPACE} --strip-components=1 -axf ${DIST_ARCHIVE}
+
+# Perform build steps in the extracted dist sources.
+
+cd ${DIST_WORKSPACE}
+FAILED=false
+
+date
+bazel fetch --distdir=${TMP_DISTDIR} "${BUILD_ONLY_TARGETS[@]}" "${TEST_TARGETS[@]}"
+
+date
+bazel build --distdir=${TMP_DISTDIR} -k \
+  "${BUILD_ONLY_TARGETS[@]}" || FAILED=true
+
+date
+bazel test --distdir=${TMP_DISTDIR} --test_output=errors -k \
+  "${TEST_TARGETS[@]}" || FAILED=true
+
+date
+cd examples
+bazel build --distdir=${TMP_DISTDIR} //... || FAILED=true
+
+if ${FAILED}; then
+   echo FAILED
+   exit 1
+fi
+echo PASS

kokoro/linux/bazel_distcheck/continuous.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh"
+timeout_mins: 15

kokoro/linux/bazel_distcheck/presubmit.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh"
+timeout_mins: 15

pkg/BUILD.bazel

@@ -4,6 +4,7 @@ load(
     "pkg_attributes",
     "pkg_filegroup",
     "pkg_files",
+    "strip_prefix",
 )
 load("//:protobuf_release.bzl", "package_naming")
 load(":build_systems.bzl", "gen_automake_file_lists", "gen_file_lists")
@@ -101,11 +102,12 @@ pkg_zip(
 pkg_filegroup(
     name = "dist_common",
     srcs = [
+        ":dist_files",
         "//:common_dist_files",
-        "//:conformance_dist_files",
         "//:cpp_dist_files",  # to build protoc
         "//benchmarks:all_dist_files",
         "//build_defs:dist_files",
+        "//conformance:all_dist_files",
         "@com_google_protobuf_examples//:dist_files",
     ],
 )
@@ -332,7 +334,7 @@ gen_automake_file_lists(
     src_libs = {
         # source rule: name in generated file
         "//:common_dist_files": "dist_common",
-        "//:conformance_dist_files": "dist_conformance",
+        "//conformance:dist_files": "dist_conformance",
         "//benchmarks:all_dist_files": "dist_benchmark",
         "@com_google_protobuf_examples//:dist_files": "dist_example",
         "//:csharp_dist_files": "dist_csharp",
@@ -374,3 +376,13 @@ cc_dist_library(
         "//:protobuf_lite",
     ],
 )
+
+################################################################################
+# Distribution sources
+################################################################################
+
+pkg_files(
+    name = "dist_files",
+    srcs = glob(["*"]),
+    strip_prefix = strip_prefix.from_root(""),
+)

third_party/BUILD.bazel

@@ -1 +1,4 @@
-exports_files(["zlib.BUILD"])
+exports_files([
+    "BUILD.bazel",
+    "zlib.BUILD",
+])