mirror of https://github.com/grpc/grpc.git
Avoid extremely slow kokoro job finalization by moving workspace to a non-synced directory on kokoro workers. (#28259)
* support "move and respawn" for kokoro jobs * delete_nonartifacts.sh no longer needed * bazel distribtest dont publish any artifacts * try the altsrc respawn for selected jobs * use altsrc for all windows CI jobs * use altsrc for all macos CI jobs * only move files under src/github * fix uploading artifacts on windowspull/28177/head
Jan Tattermusch
3 years ago
committed by
GitHub
parent
a0c975abd5
commit
3a024ea40c
34 changed files with 297 additions and 58 deletions
@ -1,27 +0,0 @@ |
||||
#!/usr/bin/env bash |
||||
# Copyright 2018 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# change to grpc repo root |
||||
cd "$(dirname "$0")/../../.." |
||||
|
||||
# After kokoro build finishes, the workspace gets rsync'ed to another machine, |
||||
# from where the artifacts and reports are processed. |
||||
# Especially on Windows, the rsync can take long time, so we cleanup the workspace |
||||
# after finishing each build. We only leave files we want to keep: |
||||
# - reports and artifacts |
||||
# - directory containing the kokoro scripts to prevent deleting a script while being executed. |
||||
time find . -type f -not -iname "*sponge_log.*" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + |
||||
@ -0,0 +1,58 @@ |
||||
@rem Copyright 2021 The gRPC Authors |
||||
@rem |
||||
@rem Licensed under the Apache License, Version 2.0 (the "License"); |
||||
@rem you may not use this file except in compliance with the License. |
||||
@rem You may obtain a copy of the License at |
||||
@rem |
||||
@rem http://www.apache.org/licenses/LICENSE-2.0 |
||||
@rem |
||||
@rem Unless required by applicable law or agreed to in writing, software |
||||
@rem distributed under the License is distributed on an "AS IS" BASIS, |
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
@rem See the License for the specific language governing permissions and |
||||
@rem limitations under the License. |
||||
|
||||
@rem Call this script at the beginning of your CI script to move |
||||
@rem the entire source tree to a different directory. |
||||
@rem That avoids slow finalization (and a number of other problems) |
||||
@rem caused by kokoro rsyncing the entire /tmpfs/src directory |
||||
@rem back to the agent. |
||||
@rem Since batch scripts don't support the "source" unix command, |
||||
@rem invoking this script is a bit trickier than on unix. |
||||
@rem This script should be invoked from the CI script like this: |
||||
@rem IF "%cd%"=="T:\src" ( |
||||
@rem call %~dp0\..\..\..\tools\internal_ci\helper_scripts\move_src_tree_and_respawn_itself.bat %0 |
||||
@rem exit /b %errorlevel% |
||||
@rem ) |
||||
|
||||
@echo off |
||||
|
||||
@rem CI script path needs to be passed as arg1. |
||||
set CI_SCRIPT_RELATIVE_TO_SRC=%1 |
||||
|
||||
@rem Check that this script was invoked under correct circumstances. |
||||
IF NOT "%cd%"=="T:\src" ( |
||||
@echo "Error: Current directory must be T:\src when invoking move_src_tree_and_respawn_itself.bat" |
||||
exit /b 1 |
||||
) |
||||
|
||||
@rem T:\ is equivalent to /tmpfs on kokoro linux. |
||||
echo "Moving workspace from T:\src to T:\altsrc and respawning the CI script." |
||||
cd /d T:\ |
||||
@rem We cannot simply rename "src" to "altsrc" as on linux since the currently running batch file is in it |
||||
@rem and windows holds a lock that prevents moving the dir. |
||||
bash -c "set -ex; mkdir -p altsrc; time cp -r src/github altsrc;" |
||||
@rem Delete files we can under the original "src/github" directory, skipping the directory that contains CI scripts |
||||
@rem (as on of the scripts is currently running and cannot be deleted) |
||||
@rem We don't want to delete files in "src" outside of "src/github" since they contain e.g kokoro input artifacts. |
||||
bash -c "set -ex; cd src/github; time find . -type f -not -path './grpc/tools/internal_ci/*' -exec rm -f {} +; ls grpc" |
||||
cd altsrc |
||||
|
||||
@rem scripts in tools/run_tests generate test reports and we need to make sure these reports |
||||
@rem land in a directory that is going to be rsynced back to the kokoro agent. |
||||
set GRPC_TEST_REPORT_BASE_DIR=T:\src\github\grpc |
||||
|
||||
echo "Invoking original CI script now." |
||||
@echo on |
||||
call "%CI_SCRIPT_RELATIVE_TO_SRC%" |
||||
exit /b %errorlevel% |
||||
@ -0,0 +1,52 @@ |
||||
#!/bin/bash |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Source this rc script at the beginning of your CI script to move |
||||
# the entire source tree to a different directory. |
||||
# That avoids slow finalization (and a number of other problems) |
||||
# caused by kokoro rsyncing the entire /tmpfs/src directory |
||||
# back to the agent. |
||||
# See b/74837748 for more context. |
||||
|
||||
set -ex |
||||
|
||||
CURRENT_DIR="$(pwd)" |
||||
CI_SCRIPT_RELATIVE_TO_SRC="$0" |
||||
|
||||
if [ "${CURRENT_DIR}" == "/tmpfs/altsrc" ] |
||||
then |
||||
# we already respawned under /tmpfs/altsrc, no need to do anything |
||||
echo "Successfully respawned the CI script ${CI_SCRIPT_RELATIVE_TO_SRC} under ${CURRENT_DIR}." |
||||
# scripts in tools/run_tests generate test reports and we need to make sure these reports |
||||
# land in a directory that is going to be rsynced back to the kokoro agent. |
||||
export GRPC_TEST_REPORT_BASE_DIR="/tmpfs/src/github/grpc" |
||||
elif [ "${CURRENT_DIR}" == "/tmpfs/src" ] |
||||
then |
||||
# we need to respawn now |
||||
# - step out of current directory |
||||
# - rename src/github to altsrc/github (/tmpfs/src is the directory that gets rsynced back to kokoro agent once the CI script finishes). |
||||
# Note that we don't want to move the entire /tmpfs/src tree (which contains e.g. the input artifacts as well, |
||||
# and their path is referenced by kokoro env variables), just /tmpfs/src/github |
||||
# which contains all the cloned github repositories (=our workspace in which the build happens) |
||||
echo "Moving workspace from /tmpfs/src to /tmpfs/altsrc and respawning the CI script." |
||||
# once "exec bash" starts, the CI script in the original location will no longer be in use and it will be safe to move |
||||
# the entire "/tmpfs/src/github" tree (moving the bash script file while running is probably safe anyway on unix, but it doesn't |
||||
# hurt to be extra careful) |
||||
exec bash -c "set -ex; cd /tmpfs; mkdir -p altsrc; mv src/github altsrc; cd altsrc; exec '${CI_SCRIPT_RELATIVE_TO_SRC}'" |
||||
else |
||||
# avoid messing with the workspace if we don't see the directory layout |
||||
# that's standard on kokoro (in case this script gets accidentally invoked outside of kokoro) |
||||
echo "Looks the script is currently not running on kokoro, skipping respawn and continuing the original CI script." |
||||
fi |
||||
@ -0,0 +1,37 @@ |
||||
#!/usr/bin/env bash |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# If you have used "move_src_tree_and_respawn_itself_rc" in a CI script |
||||
# and the script produces artifacts to be stored by kokoro, run this |
||||
# at the end of the script. |
||||
|
||||
set -ex |
||||
|
||||
if [ "${GRPC_TEST_REPORT_BASE_DIR}" == "" ] |
||||
then |
||||
# looks like the move_src_tree_and_respawn_itself_rc hasn't been used |
||||
# and we're not running under a moved source tree |
||||
exit 0 |
||||
fi |
||||
|
||||
# change to grpc repo root |
||||
cd "$(dirname "$0")/../../.." |
||||
|
||||
# If running under a moved source tree (see grpc/tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc), |
||||
# we need to copy the artifacts produced by the build to a location that's stored by kokoro. |
||||
|
||||
# artifacts in this directory will be stored by kokoro |
||||
mkdir -p "${GRPC_TEST_REPORT_BASE_DIR}/artifacts" |
||||
time cp -r artifacts/* "${GRPC_TEST_REPORT_BASE_DIR}/artifacts" || true |
||||
Loading…
Reference in new issue