The release process has been moved to an internal workflow, and all tests have been migrated to GHA PiperOrigin-RevId: 507043757pull/11815/head
Mike Kruskal
2 years ago
committed by
Copybara-Service
parent
eedef31738
commit
3becebb82a
34 changed files with 0 additions and 861 deletions
@ -1,12 +0,0 @@ |
||||
|
||||
Kokoro Infrastructure |
||||
---------------------- |
||||
|
||||
The files in this directory serve as plumbing for running Protobuf |
||||
tests under Kokoro, our internal CI. |
||||
|
||||
We have shared this part of our CI configuration in hopes that it is |
||||
helpful to contributors who want to better understand the details of |
||||
our test and release processes. If there are changes, please file an |
||||
issue; unfortunately, we may not be able to accept PRs (but feel free |
||||
to send one if it helps to explain the issue). |
||||
@ -1,44 +0,0 @@ |
||||
# Format: //devtools/kokoro/config/proto/build.proto |
||||
|
||||
# Build logs will be here |
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*sponge_log.xml" |
||||
} |
||||
} |
||||
|
||||
# Download trampoline resources. |
||||
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" |
||||
|
||||
# Use the trampoline script to run in docker. |
||||
build_file: "protobuf/kokoro/docs/trampoline.sh" |
||||
|
||||
# Configure the docker image for kokoro-trampoline. |
||||
env_vars: { |
||||
key: "TRAMPOLINE_IMAGE" |
||||
value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" |
||||
} |
||||
|
||||
env_vars: { |
||||
key: "STAGING_BUCKET" |
||||
value: "docs-staging" |
||||
} |
||||
|
||||
# Fetch the token needed for reporting release status to GitHub |
||||
before_action { |
||||
fetch_keystore { |
||||
keystore_resource { |
||||
keystore_config_id: 73713 |
||||
keyname: "yoshi-automation-github-key" |
||||
} |
||||
} |
||||
} |
||||
|
||||
before_action { |
||||
fetch_keystore { |
||||
keystore_resource { |
||||
keystore_config_id: 73713 |
||||
keyname: "docuploader_service_account" |
||||
} |
||||
} |
||||
} |
||||
@ -1,51 +0,0 @@ |
||||
#!/bin/bash |
||||
# Adapted from https://github.com/googleapis/google-cloud-python/blob/main/.kokoro/publish-docs.sh |
||||
|
||||
set -eo pipefail |
||||
|
||||
# Disable buffering, so that the logs stream through. |
||||
export PYTHONUNBUFFERED=1 |
||||
|
||||
cd github/protobuf/python |
||||
|
||||
# install package |
||||
sudo apt-get update |
||||
sudo apt-get -y install software-properties-common |
||||
sudo add-apt-repository universe |
||||
sudo apt-get update |
||||
sudo apt-get -y install unzip |
||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v21.1/protoc-21.1-linux-x86_64.zip |
||||
unzip protoc-21.1-linux-x86_64.zip bin/protoc |
||||
mv bin/protoc ../protoc |
||||
python3 -m venv venv |
||||
source venv/bin/activate |
||||
python setup.py install |
||||
|
||||
# install docs dependencies |
||||
python -m pip install -r docs/requirements.txt |
||||
|
||||
# build docs |
||||
cd docs |
||||
make html |
||||
cd .. |
||||
deactivate |
||||
|
||||
python3 -m pip install protobuf==4.21.1 gcp-docuploader==0.6.3 |
||||
|
||||
# install a json parser |
||||
sudo apt-get -y install jq |
||||
|
||||
# create metadata |
||||
python3 -m docuploader create-metadata \ |
||||
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ |
||||
--version=$(python3 setup.py --version) \ |
||||
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ |
||||
--distribution-name=$(python3 setup.py --name) \ |
||||
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ |
||||
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ |
||||
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) |
||||
|
||||
cat docs.metadata |
||||
|
||||
# upload docs |
||||
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging |
||||
@ -1,7 +0,0 @@ |
||||
# Format: //devtools/kokoro/config/proto/build.proto |
||||
|
||||
# Tell the trampoline which build file to use. |
||||
env_vars: { |
||||
key: "TRAMPOLINE_BUILD_FILE" |
||||
value: "github/protobuf/kokoro/docs/publish-python.sh" |
||||
} |
||||
@ -1,11 +0,0 @@ |
||||
#!/bin/bash |
||||
# Copied from https://github.com/googleapis/google-cloud-python/blob/main/.kokoro/trampoline.sh |
||||
|
||||
set -eo pipefail |
||||
|
||||
python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$? |
||||
|
||||
chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh |
||||
${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true |
||||
|
||||
exit ${ret_code} |
||||
@ -1,7 +0,0 @@ |
||||
build_file: "protobuf/kokoro/release/collect_all_artifacts.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "github/protobuf/artifacts/**" |
||||
} |
||||
} |
||||
@ -1,67 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# Change to repo root. |
||||
cd $(dirname $0)/../.. |
||||
|
||||
# Initialize any submodules. |
||||
git config --global --add safe.directory '*' |
||||
git submodule update --init --recursive |
||||
|
||||
# The directory with all resulting artifacts |
||||
mkdir -p artifacts |
||||
|
||||
# Artifacts from all predecessor jobs get copied to this directory by kokoro |
||||
INPUT_ARTIFACTS_DIR="${KOKORO_GFILE_DIR}/github/protobuf" |
||||
|
||||
# TODO(jtattermusch): remove listing the files, but for now it make it easier |
||||
# to iterate on the script. |
||||
ls -R ${INPUT_ARTIFACTS_DIR} |
||||
|
||||
# ==================================== |
||||
# Copy to expose all the artifacts from the predecessor jobs to the output |
||||
# TODO(jtattermusch): the directory layout of the artifact builds is pretty messy, |
||||
# so will be the output artifacts of this job. |
||||
cp -r ${INPUT_ARTIFACTS_DIR}/* artifacts |
||||
|
||||
# ==================================== |
||||
# Build Google.Protobuf.Tools C# nuget |
||||
# The reason it's being done in this script is that we need access to protoc binaries |
||||
# built on multiple platform (the build is performed by the "build artifact" step) |
||||
# and adding and extra chained build just for building the Google.Protobuf.Tools |
||||
# nuget seems like an overkill. |
||||
cd csharp |
||||
mkdir -p protoc/windows_x86 |
||||
mkdir -p protoc/windows_x64 |
||||
cp ${INPUT_ARTIFACTS_DIR}/build32/Release/protoc.exe protoc/windows_x86/protoc.exe |
||||
cp ${INPUT_ARTIFACTS_DIR}/build64/Release/protoc.exe protoc/windows_x64/protoc.exe |
||||
|
||||
mkdir -p protoc/linux_x86 |
||||
mkdir -p protoc/linux_x64 |
||||
# Because of maven unrelated reasonse the linux protoc binaries have a dummy .exe extension. |
||||
# For the Google.Protobuf.Tools nuget, we don't want that exception, so we just remove it. |
||||
cp ${INPUT_ARTIFACTS_DIR}/kokoro/release/protoc/target/linux/x86_32/protoc.exe protoc/linux_x86/protoc |
||||
cp ${INPUT_ARTIFACTS_DIR}/kokoro/release/protoc/target/linux/x86_64/protoc.exe protoc/linux_x64/protoc |
||||
|
||||
mkdir -p protoc/macosx_x64 |
||||
cp ${INPUT_ARTIFACTS_DIR}/build64/src/protoc protoc/macosx_x64/protoc |
||||
|
||||
# Install nuget (will also install mono) |
||||
# TODO(jtattermusch): use "mono:5.14" docker image instead so we don't have to apt-get install |
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
||||
echo "deb https://download.mono-project.com/repo/ubuntu stable-xenial main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list |
||||
# NVidia has stopped publishing Cuda packages for Ubuntu 16.04, so we need to |
||||
# delete this file to allow the apt update to run successfully. |
||||
sudo rm -f /etc/apt/sources.list.d/cuda.list |
||||
sudo apt update |
||||
sudo apt-get install -y nuget |
||||
|
||||
# Copy WKT protos to csharp directory. |
||||
mkdir -p src/google/protobuf |
||||
cp ../src/google/protobuf/*.proto src/google/protobuf/ |
||||
|
||||
nuget pack Google.Protobuf.Tools.nuspec |
||||
|
||||
# Copy the nupkg to the output artifacts |
||||
cp Google.Protobuf.Tools.*.nupkg ../artifacts |
||||
@ -1,17 +0,0 @@ |
||||
@rem enter repo root |
||||
cd /d %~dp0\..\..\..\.. |
||||
|
||||
cd csharp |
||||
|
||||
@rem Install dotnet SDK |
||||
powershell -File install_dotnet_sdk.ps1 |
||||
set PATH=%LOCALAPPDATA%\Microsoft\dotnet;%PATH% |
||||
|
||||
@rem Disable some unwanted dotnet options |
||||
set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true |
||||
set DOTNET_CLI_TELEMETRY_OPTOUT=true |
||||
|
||||
@rem Work around https://github.com/dotnet/core/issues/5881 |
||||
dotnet nuget locals all --clear |
||||
|
||||
call build_packages.bat |
||||
@ -1,11 +0,0 @@ |
||||
# Config file for running tests in Kokoro |
||||
|
||||
# Location of the build script in repository |
||||
build_file: "protobuf/kokoro/release/csharp/windows/build_nuget.bat" |
||||
timeout_mins: 60 |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*.nupkg" |
||||
} |
||||
} |
||||
@ -1 +0,0 @@ |
||||
# Keep this file empty! Use common.cfg instead. |
||||
@ -1 +0,0 @@ |
||||
# Keep this file empty! Use common.cfg instead. |
||||
@ -1,11 +0,0 @@ |
||||
# Config file for running tests in Kokoro |
||||
|
||||
# Location of the build script in repository |
||||
build_file: "protobuf/kokoro/release/csharp/windows/build_nuget.bat" |
||||
timeout_mins: 60 |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*.nupkg" |
||||
} |
||||
} |
||||
@ -1,249 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
# Builds protoc executable into target/<OS>/<ARCH>/protoc.exe; optionally builds |
||||
# protoc plugins into target/<OS>/<ARCH>/protoc-gen-*.exe |
||||
# |
||||
# Usage: ./build-protoc.sh <OS> <ARCH> <TARGET> |
||||
# |
||||
# <TARGET> can be "protoc" or "protoc-gen-javalite". Supported <OS> <ARCH> |
||||
# combinations: |
||||
# HOST <OS> <ARCH> <COMMENT> |
||||
# cygwin windows x86_32 Requires: i686-w64-mingw32-gcc |
||||
# cygwin windows x86_64 Requires: x86_64-w64-mingw32-gcc |
||||
# linux linux aarch_64 Requires: g++-aarch64-linux-gnu |
||||
# linux linux x86_32 |
||||
# linux linux x86_64 |
||||
# linux windows x86_32 Requires: i686-w64-mingw32-gcc |
||||
# linux windows x86_64 Requires: x86_64-w64-mingw32-gcc |
||||
# macos osx x86_32 |
||||
# macos osx x86_64 |
||||
# mingw windows x86_32 |
||||
# mingw windows x86_64 |
||||
|
||||
OS=$1 |
||||
ARCH=$2 |
||||
BAZEL_TARGET=$3 |
||||
|
||||
if [[ $# < 3 ]]; then |
||||
echo "Not enough arguments provided." |
||||
exit 1 |
||||
fi |
||||
|
||||
if [[ $BAZEL_TARGET != protoc ]]; then |
||||
echo "Target ""$BAZEL_TARGET"" invalid." |
||||
exit 1 |
||||
fi |
||||
|
||||
# Under Cygwin, bash doesn't have these in PATH when called from Maven which |
||||
# runs in Windows version of Java. |
||||
export PATH="/bin:/usr/bin:$PATH" |
||||
|
||||
############################################################################ |
||||
# Helper functions |
||||
############################################################################ |
||||
E_PARAM_ERR=98 |
||||
E_ASSERT_FAILED=99 |
||||
|
||||
# Usage: |
||||
fail() |
||||
{ |
||||
echo "ERROR: $1" |
||||
echo |
||||
exit $E_ASSERT_FAILED |
||||
} |
||||
|
||||
# Usage: assertEq VAL1 VAL2 $LINENO |
||||
assertEq () |
||||
{ |
||||
lineno=$3 |
||||
if [ -z "$lineno" ]; then |
||||
echo "lineno not given" |
||||
exit $E_PARAM_ERR |
||||
fi |
||||
|
||||
if [[ "$1" != "$2" ]]; then |
||||
echo "Assertion failed: \"$1\" == \"$2\"" |
||||
echo "File \"$0\", line $lineno" # Give name of file and line number. |
||||
exit $E_ASSERT_FAILED |
||||
fi |
||||
} |
||||
|
||||
# Checks the artifact is for the expected architecture |
||||
# Usage: checkArch <path-to-protoc> |
||||
checkArch () |
||||
{ |
||||
echo |
||||
echo "Checking file format ..." |
||||
if [[ "$OS" == windows || "$OS" == linux ]]; then |
||||
format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")" |
||||
echo Format=$format |
||||
if [[ "$OS" == linux ]]; then |
||||
host_machine="$(uname -m)"; |
||||
if [[ "$ARCH" == x86_32 ]]; then |
||||
assertEq $format "elf32-i386" $LINENO |
||||
elif [[ "$ARCH" == x86_64 ]]; then |
||||
assertEq $format "elf64-x86-64" $LINENO |
||||
elif [[ "$ARCH" == aarch_64 ]]; then |
||||
assertEq $format "elf64-little" $LINENO |
||||
elif [[ "$ARCH" == s390_64 ]]; then |
||||
if [[ $host_machine == s390x ]];then |
||||
assertEq $format "elf64-s390" $LINENO |
||||
else |
||||
assertEq $format "elf64-big" $LINENO |
||||
fi |
||||
elif [[ "$ARCH" == ppcle_64 ]]; then |
||||
if [[ $host_machine == ppc64le ]];then |
||||
assertEq $format "elf64-powerpcle" $LINENO |
||||
else |
||||
assertEq $format "elf64-little" $LINENO |
||||
fi |
||||
else |
||||
fail "Unsupported arch: $ARCH" |
||||
fi |
||||
else |
||||
# $OS == windows |
||||
if [[ "$ARCH" == x86_32 ]]; then |
||||
assertEq $format "pei-i386" $LINENO |
||||
elif [[ "$ARCH" == x86_64 ]]; then |
||||
assertEq $format "pei-x86-64" $LINENO |
||||
else |
||||
fail "Unsupported arch: $ARCH" |
||||
fi |
||||
fi |
||||
elif [[ "$OS" == osx ]]; then |
||||
format="$(file -b "$1" | grep -o "[^ ]*$")" |
||||
echo Format=$format |
||||
if [[ "$ARCH" == x86_32 ]]; then |
||||
assertEq $format "i386" $LINENO |
||||
elif [[ "$ARCH" == x86_64 ]]; then |
||||
assertEq $format "x86_64" $LINENO |
||||
else |
||||
fail "Unsupported arch: $ARCH" |
||||
fi |
||||
else |
||||
fail "Unsupported system: $OS" |
||||
fi |
||||
echo |
||||
} |
||||
|
||||
# Checks the dependencies of the artifact. Artifacts should only depend on |
||||
# system libraries. |
||||
# Usage: checkDependencies <path-to-protoc> |
||||
checkDependencies () |
||||
{ |
||||
if [[ "$OS" == windows ]]; then |
||||
dump_cmd='objdump -x '"$1"' | fgrep "DLL Name"' |
||||
white_list="KERNEL32\.dll\|msvcrt\.dll" |
||||
elif [[ "$OS" == linux ]]; then |
||||
host_machine="$(uname -m)"; |
||||
dump_cmd='ldd '"$1" |
||||
if [[ "$ARCH" == x86_32 ]]; then |
||||
white_list="linux-gate\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libgcc_s\.so\.1\|libstdc++\.so\.6\|ld-linux\.so\.2" |
||||
elif [[ "$ARCH" == x86_64 ]]; then |
||||
white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libgcc_s\.so\.1\|libstdc++\.so\.6\|ld-linux-x86-64\.so\.2" |
||||
elif [[ "$ARCH" == s390_64 ]]; then |
||||
if [[ $host_machine != s390x ]];then |
||||
dump_cmd='objdump -p '"$1"' | grep NEEDED' |
||||
fi |
||||
white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libgcc_s\.so\.1\|libstdc++\.so\.6\|libz\.so\.1\|ld64\.so\.1" |
||||
elif [[ "$ARCH" == ppcle_64 ]]; then |
||||
if [[ $host_machine != ppc64le ]];then |
||||
dump_cmd='objdump -p '"$1"' | grep NEEDED' |
||||
fi |
||||
white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libgcc_s\.so\.1\|libstdc++\.so\.6\|libz\.so\.1\|ld64\.so\.2" |
||||
elif [[ "$ARCH" == aarch_64 ]]; then |
||||
dump_cmd='objdump -p '"$1"' | grep NEEDED' |
||||
white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libgcc_s\.so\.1\|libstdc++\.so\.6\|ld-linux-aarch64\.so\.1" |
||||
fi |
||||
elif [[ "$OS" == osx ]]; then |
||||
dump_cmd='otool -L '"$1"' | fgrep dylib' |
||||
white_list="libz\.1\.dylib\|libstdc++\.6\.dylib\|libSystem\.B\.dylib" |
||||
fi |
||||
if [[ -z "$white_list" || -z "$dump_cmd" ]]; then |
||||
fail "Unsupported platform $OS-$ARCH." |
||||
fi |
||||
echo "Checking for expected dependencies ..." |
||||
eval $dump_cmd | grep -i "$white_list" || fail "doesn't show any expected dependencies" |
||||
echo "Checking for unexpected dependencies ..." |
||||
eval $dump_cmd | grep -i -v "$white_list" |
||||
ret=$? |
||||
if [[ $ret == 0 ]]; then |
||||
fail "found unexpected dependencies (listed above)." |
||||
elif [[ $ret != 1 ]]; then |
||||
fail "Error when checking dependencies." |
||||
fi # grep returns 1 when "not found", which is what we expect |
||||
echo "Dependencies look good." |
||||
echo |
||||
} |
||||
############################################################################ |
||||
|
||||
echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$BAZEL_TARGET" |
||||
|
||||
BAZEL_ARGS=("--dynamic_mode=off" "--compilation_mode=opt" "--copt=-g0" "--copt=-fpic") |
||||
|
||||
if [[ "$OS" == windows ]]; then |
||||
BAZEL_TARGET="${BAZEL_TARGET}.exe" |
||||
fi |
||||
|
||||
if [[ "$(uname)" == CYGWIN* ]]; then |
||||
assertEq "$OS" windows $LINENO |
||||
# Use mingw32 compilers because executables produced by Cygwin compiler |
||||
# always have dependency on Cygwin DLL. |
||||
if [[ "$ARCH" == x86_64 ]]; then |
||||
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" |
||||
elif [[ "$ARCH" == x86_32 ]]; then |
||||
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-pc-mingw32" |
||||
else |
||||
fail "Unsupported arch by CYGWIN: $ARCH" |
||||
fi |
||||
elif [[ "$(uname)" == MINGW32* ]]; then |
||||
assertEq "$OS" windows $LINENO |
||||
assertEq "$ARCH" x86_32 $LINENO |
||||
elif [[ "$(uname)" == MINGW64* ]]; then |
||||
assertEq "$OS" windows $LINENO |
||||
assertEq "$ARCH" x86_64 $LINENO |
||||
elif [[ "$(uname)" == Linux* ]]; then |
||||
if [[ "$OS" == linux ]]; then |
||||
BAZEL_ARGS+=("--config=linux-$ARCH") |
||||
elif [[ "$OS" == windows ]]; then |
||||
# Cross-compilation for Windows |
||||
if [[ "$ARCH" == x86_64 ]]; then |
||||
BAZEL_ARGS+=("--config=win64") |
||||
elif [[ "$ARCH" == x86_32 ]]; then |
||||
BAZEL_ARGS+=("--config=win32") |
||||
else |
||||
fail "Unsupported arch: $ARCH" |
||||
fi |
||||
else |
||||
fail "Cannot build $OS on $(uname)" |
||||
fi |
||||
elif [[ "$(uname)" == Darwin* ]]; then |
||||
assertEq "$OS" osx $LINENO |
||||
# Make the binary compatible with OSX 10.7 and later |
||||
BAZEL_ARGS+=("--config=osx-$ARCH" "--macos_minimum_os=10.7") |
||||
else |
||||
fail "Unsupported system: $(uname)" |
||||
fi |
||||
|
||||
# Nested double quotes are unintuitive, but it works. |
||||
cd "$(dirname "$0")" |
||||
|
||||
WORKING_DIR="$(pwd)" |
||||
TARGET_FILE="target/$OS/$ARCH/$BAZEL_TARGET.exe" |
||||
DOCKER_IMAGE=us-docker.pkg.dev/protobuf-build/release-containers/linux@sha256:85fd92cce31eb9446ca85d108b424b8167cea3469b58f2b9a9d30e37fc467a00 |
||||
gcloud components update --quiet |
||||
gcloud auth configure-docker us-docker.pkg.dev --quiet |
||||
|
||||
tmpfile=$(mktemp -u) && |
||||
docker run --cidfile $tmpfile -v $WORKING_DIR/../../..:/workspace $DOCKER_IMAGE \ |
||||
build //:$BAZEL_TARGET "${BAZEL_ARGS[@]}" && |
||||
mkdir -p $(dirname $TARGET_FILE) && |
||||
docker cp \ |
||||
`cat $tmpfile`:/workspace/bazel-bin/$BAZEL_TARGET $TARGET_FILE || exit 1 |
||||
|
||||
if [[ "$OS" == osx ]]; then |
||||
# Since Mac linker doesn't accept "-s", we need to run strip |
||||
strip $TARGET_FILE || exit 1 |
||||
fi |
||||
|
||||
checkArch $TARGET_FILE && checkDependencies $TARGET_FILE |
||||
@ -1,29 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
# This is not the source of truth for release protoc executables, and will soon |
||||
# be deprecated. |
||||
|
||||
set -ex |
||||
|
||||
# Change to repo root. |
||||
cd $(dirname $0)/../../../.. |
||||
GIT_REPO_ROOT=$(pwd) |
||||
|
||||
# Initialize any submodules. |
||||
git submodule update --init --recursive |
||||
|
||||
# Cross-build for aarch64, ppc64le and s390x. Note: we do these builds first to avoid |
||||
# file permission issues. The Docker builds will create directories owned by |
||||
# root, which causes problems if we try to add new artifacts to those |
||||
# directories afterward. |
||||
|
||||
|
||||
kokoro/release/protoc/build-protoc.sh linux aarch_64 protoc |
||||
|
||||
kokoro/release/protoc/build-protoc.sh linux ppcle_64 protoc |
||||
|
||||
kokoro/release/protoc/build-protoc.sh linux s390_64 protoc |
||||
|
||||
kokoro/release/protoc/build-protoc.sh linux x86_64 protoc |
||||
|
||||
kokoro/release/protoc/build-protoc.sh linux x86_32 protoc |
||||
@ -1,7 +0,0 @@ |
||||
build_file: "protobuf/kokoro/release/protoc/linux/build.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "**/protoc.exe" |
||||
} |
||||
} |
||||
@ -1,12 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
CXXFLAGS_COMMON="-std=c++14 -DNDEBUG -mmacosx-version-min=10.9" |
||||
|
||||
cd github/protobuf |
||||
|
||||
bazel build //:protoc --dynamic_mode=off |
||||
mkdir -p build64/src |
||||
cp bazel-bin/protoc build64/src/protoc |
||||
file bazel-bin/protoc |
||||
otool -L bazel-bin/protoc | grep dylib |
||||
@ -1,8 +0,0 @@ |
||||
# Configuration for Mac OSX protoc release builds |
||||
build_file: "protobuf/kokoro/release/protoc/macos/build.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "**/protoc" |
||||
} |
||||
} |
||||
@ -1,62 +0,0 @@ |
||||
set generator32=Visual Studio 15 |
||||
set generator64=Visual Studio 15 Win64 |
||||
set vcplatform32=win32 |
||||
set vcplatform64=x64 |
||||
set configuration=Release |
||||
|
||||
echo Building protoc |
||||
cd github\protobuf |
||||
|
||||
@rem Update Chocolatey |
||||
choco upgrade -y --no-progress chocolatey |
||||
choco install -y python --version 3.10.0 |
||||
choco install -y --no-progress --pre cmake |
||||
|
||||
@rem Enable long paths. |
||||
Powershell.exe -Command "New-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1 -PropertyType DWORD -Force" |
||||
|
||||
@rem Update git submodules. |
||||
git submodule update --init --recursive |
||||
|
||||
@rem Select Visual Studio 2017. |
||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x64 |
||||
|
||||
@rem Convert Windows line breaks to Unix line breaks |
||||
@rem This allows text-matching tests to pass |
||||
@find . -type f -print0 | xargs -0 d2u |
||||
|
||||
@rem Use python3 |
||||
C:\python310\python.exe -m venv venv |
||||
call venv\Scripts\activate.bat |
||||
|
||||
@rem Allow Bazel to create short paths. |
||||
fsutil 8dot3name set 0 |
||||
|
||||
@rem Reinstall Bazel due to corrupt installation in kokoro. |
||||
bazel version |
||||
choco install bazel -y -i --version 5.1.0 |
||||
bazel version |
||||
|
||||
@rem Regenerate stale files. |
||||
bazel test //src:cmake_lists_staleness_test || call bazel-bin\src\cmake_lists_staleness_test.exe --fix |
||||
bazel test //src/google/protobuf:well_known_types_staleness_test || call bazel-bin\src\google\protobuf\well_known_types_staleness_test.exe --fix |
||||
|
||||
set ABSL_ROOT_DIR=%cd%\third_party\abseil-cpp |
||||
|
||||
mkdir build32 |
||||
cd build32 |
||||
cmake -G "%generator32%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON .. |
||||
msbuild protobuf.sln /p:Platform=%vcplatform32% || goto error |
||||
cd .. |
||||
|
||||
mkdir build64 |
||||
cd build64 |
||||
cmake -G "%generator64%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON .. |
||||
msbuild protobuf.sln /p:Platform=%vcplatform64% || goto error |
||||
cd .. |
||||
|
||||
goto :EOF |
||||
|
||||
:error |
||||
echo Failed! |
||||
exit /b %ERRORLEVEL% |
||||
@ -1,8 +0,0 @@ |
||||
# Configuration for Windows protoc release builds |
||||
build_file: "protobuf/kokoro/release/protoc/windows/build.bat" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "**/protoc.exe" |
||||
} |
||||
} |
||||
@ -1,14 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# change to repo root |
||||
cd $(dirname $0)/../../../.. |
||||
|
||||
source kokoro/release/ruby/linux/prepare_build.sh |
||||
|
||||
# ruby environment |
||||
source kokoro/release/ruby/linux/ruby/ruby_build_environment.sh |
||||
|
||||
# build artifacts |
||||
bash kokoro/release/ruby/linux/ruby/ruby_build.sh |
||||
@ -1,8 +0,0 @@ |
||||
# Configuration for Linux release builds |
||||
build_file: "protobuf/kokoro/release/ruby/linux/build_artifacts.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "github/protobuf/artifacts/**" |
||||
} |
||||
} |
||||
@ -1 +0,0 @@ |
||||
# Keep this file empty! Use common.cfg instead. |
||||
@ -1,12 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
# Move docker's storage location to scratch disk so we don't run out of space. |
||||
echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /etc/default/docker |
||||
# Use container registry mirror for pulling docker images (should make downloads faster) |
||||
# See https://cloud.google.com/container-registry/docs/using-dockerhub-mirroring |
||||
echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker |
||||
sudo service docker restart |
||||
|
||||
# All artifacts come here |
||||
mkdir artifacts |
||||
export ARTIFACT_DIR=$(pwd)/artifacts |
||||
@ -1 +0,0 @@ |
||||
# Keep this file empty! Use common.cfg instead. |
||||
@ -1,8 +0,0 @@ |
||||
# Configuration for Linux release builds |
||||
build_file: "protobuf/kokoro/release/ruby/linux/build_artifacts.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "github/protobuf/artifacts/**" |
||||
} |
||||
} |
||||
@ -1,22 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# Build protoc |
||||
use_bazel.sh 5.1.1 |
||||
bazel build //:protoc |
||||
|
||||
# The java build setup expects protoc in the root directory. |
||||
cp bazel-bin/protoc . |
||||
export PROTOC=$PWD/protoc |
||||
|
||||
# Pull in dependencies. |
||||
git submodule update --init --recursive |
||||
|
||||
umask 0022 |
||||
pushd ruby |
||||
gem install bundler -v 2.1.4 |
||||
bundle update && bundle exec rake gem:native |
||||
ls pkg |
||||
mv pkg/* $ARTIFACT_DIR |
||||
popd |
||||
@ -1,9 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set +ex |
||||
[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh |
||||
set -e # rvm commands are very verbose |
||||
rvm --default use ruby-2.4.1 |
||||
# The version needs to be updated if the version specified in Gemfile.lock is changed |
||||
gem install bundler -v '1.17.3' |
||||
set -ex |
||||
@ -1,16 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# change to repo root |
||||
cd $(dirname $0)/../../../.. |
||||
|
||||
# all artifacts come here |
||||
mkdir artifacts |
||||
export ARTIFACT_DIR=$(pwd)/artifacts |
||||
|
||||
# ruby environment |
||||
bash kokoro/release/ruby/macos/ruby/ruby_build_environment.sh |
||||
|
||||
# build artifacts |
||||
bash kokoro/release/ruby/macos/ruby/ruby_build.sh |
||||
@ -1,8 +0,0 @@ |
||||
# Configuration for Mac OSX release builds |
||||
build_file: "protobuf/kokoro/release/ruby/macos/build_artifacts.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "github/protobuf/artifacts/**" |
||||
} |
||||
} |
||||
@ -1 +0,0 @@ |
||||
# Keep this file empty! Use common.cfg instead. |
||||
@ -1 +0,0 @@ |
||||
# Keep this file empty! Use common.cfg instead. |
||||
@ -1,8 +0,0 @@ |
||||
# Configuration for Mac OSX release builds |
||||
build_file: "protobuf/kokoro/release/ruby/macos/build_artifacts.sh" |
||||
|
||||
action { |
||||
define_artifacts { |
||||
regex: "github/protobuf/artifacts/**" |
||||
} |
||||
} |
||||
@ -1,17 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# Build protoc |
||||
bazel build //:protoc |
||||
export PROTOC=$PWD/bazel-bin/protoc |
||||
|
||||
# Pull in dependencies. |
||||
git submodule update --init --recursive |
||||
|
||||
umask 0022 |
||||
pushd ruby |
||||
bundle update && bundle exec rake gem:native |
||||
ls pkg |
||||
mv pkg/* $ARTIFACT_DIR |
||||
popd |
||||
@ -1,120 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# Fix permissions |
||||
sudo chown -R $(whoami) $HOME/.rvm/ |
||||
sudo chown -R $(whoami) /Library/Ruby/ |
||||
|
||||
set +ex # rvm script is very verbose and exits with errorcode |
||||
|
||||
# Fix permissions |
||||
sudo chown -R $(whoami) $HOME/.rvm/ |
||||
sudo chown -R $(whoami) /Library/Ruby/ |
||||
|
||||
source $HOME/.rvm/scripts/rvm |
||||
set -e # rvm commands are very verbose |
||||
time rvm install 2.6.0 |
||||
rvm use 2.6.0 |
||||
gem install rake-compiler --no-document |
||||
gem install bundler --no-document |
||||
time rvm install 3.1.0 |
||||
rvm use 3.1.0 |
||||
gem install rake-compiler --no-document |
||||
gem install bundler --no-document |
||||
time rvm install 2.7.0 |
||||
rvm use 2.7.0 --default |
||||
gem install rake-compiler --no-document |
||||
gem install bundler --no-document |
||||
rvm osx-ssl-certs status all |
||||
rvm osx-ssl-certs update all |
||||
set -ex |
||||
|
||||
rm -rf ~/.rake-compiler |
||||
|
||||
CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) |
||||
CROSS_RUBY31=$(mktemp tmpfile.XXXXXXXX) |
||||
|
||||
|
||||
curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/72184e51779b6a3b9b8580b036a052fdc3181ced/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" |
||||
|
||||
# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details |
||||
patch "$CROSS_RUBY" << EOF |
||||
--- cross-ruby.rake 2020-12-11 11:17:53.000000000 +0900 |
||||
+++ patched 2020-12-11 11:18:52.000000000 +0900 |
||||
@@ -111,10 +111,12 @@ |
||||
"--host=#{MINGW_HOST}", |
||||
"--target=#{MINGW_TARGET}", |
||||
"--build=#{RUBY_BUILD}", |
||||
- '--enable-shared', |
||||
+ '--enable-static', |
||||
+ '--disable-shared', |
||||
'--disable-install-doc', |
||||
+ '--without-gmp', |
||||
'--with-ext=', |
||||
- 'LDFLAGS=-pipe -s', |
||||
+ 'LDFLAGS=-pipe', |
||||
] |
||||
|
||||
# Force Winsock2 for Ruby 1.8, 1.9 defaults to it |
||||
@@ -130,6 +132,7 @@ |
||||
# make |
||||
file "#{build_dir}/ruby.exe" => ["#{build_dir}/Makefile"] do |t| |
||||
chdir File.dirname(t.prerequisites.first) do |
||||
+ sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make |
||||
sh MAKE |
||||
end |
||||
end |
||||
EOF |
||||
|
||||
cp $CROSS_RUBY $CROSS_RUBY31 |
||||
|
||||
patch "$CROSS_RUBY31" << EOF |
||||
--- cross-ruby.rake 2022-03-04 11:49:52.000000000 +0000 |
||||
+++ patched 2022-03-04 11:58:22.000000000 +0000 |
||||
@@ -114,6 +114,7 @@ |
||||
'--enable-static', |
||||
'--disable-shared', |
||||
'--disable-install-doc', |
||||
+ '--with-coroutine=ucontext', |
||||
'--without-gmp', |
||||
'--with-ext=', |
||||
'LDFLAGS=-pipe', |
||||
EOF |
||||
|
||||
MAKE="make -j8" |
||||
|
||||
set +x # rvm commands are very verbose |
||||
rvm use 3.1.0 |
||||
set -x |
||||
ruby --version | grep 'ruby 3.1.0' |
||||
for v in 3.1.0 ; do |
||||
ccache -c |
||||
rake -f "$CROSS_RUBY31" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" |
||||
# Disabled until it can be fixed: https://github.com/protocolbuffers/protobuf/issues/9804 |
||||
# rake -f "$CROSS_RUBY31" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE" |
||||
done |
||||
|
||||
set +x # rvm commands are very verbose |
||||
rvm use 2.7.0 |
||||
set -x |
||||
ruby --version | grep 'ruby 2.7.0' |
||||
for v in 3.0.0 2.7.0 ; do |
||||
ccache -c |
||||
rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" |
||||
# Disabled until it can be fixed: https://github.com/protocolbuffers/protobuf/issues/9804 |
||||
# rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE" |
||||
done |
||||
set +x |
||||
rvm use 2.6.0 |
||||
set -x |
||||
ruby --version | grep 'ruby 2.6.0' |
||||
for v in 2.6.0 ; do |
||||
ccache -c |
||||
rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" |
||||
# Disabled until it can be fixed: https://github.com/protocolbuffers/protobuf/issues/9804 |
||||
# rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE" |
||||
done |
||||
set +x |
||||
rvm use 2.7.0 |
||||
set -x |
||||
Loading…
Reference in new issue