diff --git a/BUILD.bazel b/BUILD.bazel index ae50fa6dc6..044e01e4f5 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1284,6 +1284,7 @@ pkg_files( "generate_descriptor_proto.sh", "maven_install.json", "update_file_lists.sh", + "//third_party:BUILD.bazel", "//third_party:zlib.BUILD", "//util/python:BUILD.bazel", ], diff --git a/benchmarks/BUILD.bazel b/benchmarks/BUILD.bazel index 83b2c7ee59..36f6283f6c 100644 --- a/benchmarks/BUILD.bazel +++ b/benchmarks/BUILD.bazel @@ -75,7 +75,6 @@ pkg_files( ], exclude = [ "__init__.py", # not in autotools dist - "BUILD.bazel", "go/*", ], ), diff --git a/benchmarks/cpp/BUILD.bazel b/benchmarks/cpp/BUILD.bazel index ba93f641e5..7a3d3ba63e 100644 --- a/benchmarks/cpp/BUILD.bazel +++ b/benchmarks/cpp/BUILD.bazel @@ -17,7 +17,10 @@ cc_binary( pkg_files( name = "dist_files", - srcs = ["cpp_benchmark.cc"], + srcs = [ + "BUILD.bazel", + "cpp_benchmark.cc", + ], strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message1/proto2/BUILD.bazel b/benchmarks/datasets/google_message1/proto2/BUILD.bazel index e3159327b9..30caed574e 100644 --- a/benchmarks/datasets/google_message1/proto2/BUILD.bazel +++ b/benchmarks/datasets/google_message1/proto2/BUILD.bazel @@ -46,7 +46,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message1/proto3/BUILD.bazel b/benchmarks/datasets/google_message1/proto3/BUILD.bazel index 2628d020aa..0dc59d1445 100644 --- a/benchmarks/datasets/google_message1/proto3/BUILD.bazel +++ b/benchmarks/datasets/google_message1/proto3/BUILD.bazel @@ -46,7 +46,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message2/BUILD.bazel b/benchmarks/datasets/google_message2/BUILD.bazel index a3208d08c3..f3d66a25c0 100644 --- a/benchmarks/datasets/google_message2/BUILD.bazel +++ b/benchmarks/datasets/google_message2/BUILD.bazel @@ -46,7 +46,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message3/BUILD.bazel b/benchmarks/datasets/google_message3/BUILD.bazel index 130c8749b3..a729e50fbc 100644 --- a/benchmarks/datasets/google_message3/BUILD.bazel +++ b/benchmarks/datasets/google_message3/BUILD.bazel @@ -52,7 +52,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message4/BUILD.bazel b/benchmarks/datasets/google_message4/BUILD.bazel index 9c7190d93b..33de093788 100644 --- a/benchmarks/datasets/google_message4/BUILD.bazel +++ b/benchmarks/datasets/google_message4/BUILD.bazel @@ -47,7 +47,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index 2b30604484..7de2b8e649 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -1,7 +1,13 @@ # Conformance testing for Protobuf. -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library", "cc_proto_library") -load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library") +load( + "@rules_pkg//:mappings.bzl", + "pkg_attributes", + "pkg_filegroup", + "pkg_files", + "strip_prefix", +) exports_files([ "conformance_test_runner.sh", @@ -145,16 +151,33 @@ pkg_files( srcs = glob( ["**/*"], exclude = [ + # Handled by dist_scripts: + "conformance_test_runner.sh", + # The following are not in autotools dist: - "conformance/autoload.php", - "conformance/conformance_nodejs.js", - "conformance/conformance_test_runner.sh", - "conformance/failure_list_java_lite.txt", - "conformance/failure_list_jruby.txt", - "conformance/text_format_failure_list_*.txt", - "conformance/update_failure_list.py", + "autoload.php", + "conformance_nodejs.js", + "failure_list_jruby.txt", + "update_failure_list.py", ], ), strip_prefix = strip_prefix.from_root(""), visibility = ["//pkg:__pkg__"], ) + +pkg_files( + name = "dist_scripts", + srcs = ["conformance_test_runner.sh"], + attributes = pkg_attributes(mode = "0555"), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//pkg:__pkg__"], +) + +pkg_filegroup( + name = "all_dist_files", + srcs = [ + ":dist_files", + ":dist_scripts", + ], + visibility = ["//pkg:__pkg__"], +) diff --git a/kokoro/linux/bazel_distcheck/build.sh b/kokoro/linux/bazel_distcheck/build.sh new file mode 100755 index 0000000000..4eb6e11264 --- /dev/null +++ b/kokoro/linux/bazel_distcheck/build.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Build file to set up and run tests using bazel-build dist archive +# +# Note that the builds use WORKSPACE to fetch external sources, not +# git submodules. + +set -eux + +BUILD_ONLY_TARGETS=( + //pkg:all + //:protoc + //:protobuf + //:protobuf_python +) + +TEST_TARGETS=( + //build_defs:all + //conformance:all + //java:tests + //:protobuf_test +) + +use_bazel.sh 5.0.0 || true +bazel version + +# Change to repo root +cd $(dirname $0)/../../.. + +# Construct temp directory for running the dist build. +# If you want to run locally and keep the build dir, create a directory +# and pass it in the DIST_WORK_ROOT env var. +if [[ -z ${DIST_WORK_ROOT:-} ]]; then + : ${DIST_WORK_ROOT:=$(mktemp -d)} + function dist_cleanup() { + rm -rf ${DIST_WORK_ROOT} + } + trap dist_cleanup EXIT +fi + +# Let Bazel share the distdir. +TMP_DISTDIR=${DIST_WORK_ROOT}/bazel-distdir +mkdir -p ${TMP_DISTDIR} + +# Build distribution archive +date +bazel fetch --distdir=${TMP_DISTDIR} //pkg:dist_all_tar +bazel build --distdir=${TMP_DISTDIR} //pkg:dist_all_tar +DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz) +bazel shutdown + +# The `pkg_tar` rule emits a symlink based on the rule name. The actual +# file is named with the current version. +date +echo "Resolved archive path: ${DIST_ARCHIVE}" + +# Extract the dist archive. +date +DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf +mkdir -p ${DIST_WORKSPACE} +tar -C ${DIST_WORKSPACE} --strip-components=1 -axf ${DIST_ARCHIVE} + +# Perform build steps in the extracted dist sources. + +cd ${DIST_WORKSPACE} +FAILED=false + +date +bazel fetch --distdir=${TMP_DISTDIR} "${BUILD_ONLY_TARGETS[@]}" "${TEST_TARGETS[@]}" + +date +bazel build --distdir=${TMP_DISTDIR} -k \ + "${BUILD_ONLY_TARGETS[@]}" || FAILED=true + +date +bazel test --distdir=${TMP_DISTDIR} --test_output=errors -k \ + "${TEST_TARGETS[@]}" || FAILED=true + +date +cd examples +bazel build --distdir=${TMP_DISTDIR} //... || FAILED=true + +if ${FAILED}; then + echo FAILED + exit 1 +fi +echo PASS diff --git a/kokoro/linux/bazel_distcheck/continuous.cfg b/kokoro/linux/bazel_distcheck/continuous.cfg new file mode 100644 index 0000000000..4ea8b21006 --- /dev/null +++ b/kokoro/linux/bazel_distcheck/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" +timeout_mins: 15 diff --git a/kokoro/linux/bazel_distcheck/presubmit.cfg b/kokoro/linux/bazel_distcheck/presubmit.cfg new file mode 100644 index 0000000000..4ea8b21006 --- /dev/null +++ b/kokoro/linux/bazel_distcheck/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" +timeout_mins: 15 diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 271810f66f..0de27d3a27 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -4,6 +4,7 @@ load( "pkg_attributes", "pkg_filegroup", "pkg_files", + "strip_prefix", ) load("//:protobuf_release.bzl", "package_naming") load(":build_systems.bzl", "gen_automake_file_lists", "gen_file_lists") @@ -101,11 +102,12 @@ pkg_zip( pkg_filegroup( name = "dist_common", srcs = [ + ":dist_files", "//:common_dist_files", - "//:conformance_dist_files", "//:cpp_dist_files", # to build protoc "//benchmarks:all_dist_files", "//build_defs:dist_files", + "//conformance:all_dist_files", "@com_google_protobuf_examples//:dist_files", ], ) @@ -332,7 +334,7 @@ gen_automake_file_lists( src_libs = { # source rule: name in generated file "//:common_dist_files": "dist_common", - "//:conformance_dist_files": "dist_conformance", + "//conformance:dist_files": "dist_conformance", "//benchmarks:all_dist_files": "dist_benchmark", "@com_google_protobuf_examples//:dist_files": "dist_example", "//:csharp_dist_files": "dist_csharp", @@ -374,3 +376,13 @@ cc_dist_library( "//:protobuf_lite", ], ) + +################################################################################ +# Distribution sources +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["*"]), + strip_prefix = strip_prefix.from_root(""), +) diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel index a8b35efceb..d12b3e37f7 100644 --- a/third_party/BUILD.bazel +++ b/third_party/BUILD.bazel @@ -1 +1,4 @@ -exports_files(["zlib.BUILD"]) +exports_files([ + "BUILD.bazel", + "zlib.BUILD", +])