delete no-longer-needed scripts from tools/jenkins

pull/15424/head
Jan Tattermusch 7 years ago
parent ce0faaf323
commit a5312c1898
  1. 7
      tools/jenkins/README.md
  2. 27
      tools/jenkins/build_artifacts.sh
  3. 27
      tools/jenkins/build_packages.sh
  4. 23
      tools/jenkins/reboot_worker.sh
  5. 34
      tools/jenkins/run_distribtest.sh
  6. 29
      tools/jenkins/run_full_cloud_prod.sh
  7. 57
      tools/jenkins/run_full_performance.sh
  8. 58
      tools/jenkins/run_full_performance_released.sh
  9. 33
      tools/jenkins/run_fuzzer.sh
  10. 24
      tools/jenkins/run_interop.sh
  11. 24
      tools/jenkins/run_interop_objc.sh
  12. 55
      tools/jenkins/run_jenkins.sh
  13. 27
      tools/jenkins/run_jenkins_matrix.sh
  14. 21
      tools/jenkins/run_line_count.sh
  15. 26
      tools/jenkins/run_performance.sh
  16. 50
      tools/jenkins/run_portability.sh
  17. 23
      tools/jenkins/run_qps_diff.sh
  18. 34
      tools/jenkins/run_sweep_performance.sh
  19. 23
      tools/jenkins/run_trickle_diff.sh

@ -1,6 +1,3 @@
# Jenkins CI scripts
# (Deprecated) Jenkins CI scripts
Scripts invoked by Jenkins (our CI platform) to run gRPC test suites.
We run a comprehensive set of tests (unit, integration, interop,
performance, portability..) on each pull request and also periodically on
`master` and release branches.
We are no longer using Jenkins to run our tests. This directory is scheduled for removal.

@ -1,27 +0,0 @@
#!/usr/bin/env bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and triggers build of artifacts.
#
# To prevent cygwin bash complaining about empty lines ending with \r
# we set the igncr option. The option doesn't exist on Linux, so we fallback
# to just 'set -ex' there.
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
curr_platform="$platform"
unset platform # variable named 'platform' breaks the windows build
python tools/run_tests/task_runner.py -f artifact $language $curr_platform $architecture

@ -1,27 +0,0 @@
#!/usr/bin/env bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and triggers build of artifacts.
#
# To prevent cygwin bash complaining about empty lines ending with \r
# we set the igncr option. The option doesn't exist on Linux, so we fallback
# to just 'set -ex' there.
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
curr_platform="$platform"
unset platform # variable named 'platform' breaks the windows build
python tools/run_tests/task_runner.py -f package $curr_platform

@ -1,23 +0,0 @@
#!/usr/bin/env bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Reboots Jenkins worker
#
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
# Give 5 seconds to finish the current job, then kill the jenkins slave process
# to avoid running any other jobs on the worker and restart the worker.
nohup sh -c 'sleep 5; killall java; sudo reboot' &

@ -1,34 +0,0 @@
#!/usr/bin/env bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and triggers run of distribution tests.
#
# To prevent cygwin bash complaining about empty lines ending with \r
# we set the igncr option. The option doesn't exist on Linux, so we fallback
# to just 'set -ex' there.
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
curr_platform="$platform"
unset platform # variable named 'platform' breaks the windows build
# Try collecting the artifacts to test from previous Jenkins build step
mkdir -p input_artifacts
cp -r platform=windows/artifacts/* input_artifacts || true
cp -r platform=linux/artifacts/* input_artifacts || true
python tools/run_tests/task_runner.py -j 4 \
-f distribtest $language $curr_platform $architecture \
$@

@ -1,29 +0,0 @@
#!/usr/bin/env bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs interop test suite.
set -ex
export LANG=en_US.UTF-8
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/run_interop_tests.py \
-l all \
--cloud_to_prod \
--cloud_to_prod_auth \
--prod_servers default cloud_gateway gateway_v4 cloud_gateway_v4 \
--use_docker -t -j 12 $@ || true

@ -1,57 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs full performance test suite.
set -ex
# Enter the gRPC repo root
cd $(dirname $0)/../..
# run 8core client vs 8core server
tools/run_tests/run_performance_tests.py \
-l c++ csharp ruby java python go php7 php7_protobuf_c \
--netperf \
--category scalable \
--bq_result_table performance_test.performance_experiment \
--remote_worker_host grpc-performance-server-8core grpc-performance-client-8core grpc-performance-client2-8core \
--xml_report report_8core.xml \
|| EXIT_CODE=1
# prevent pushing leftover build files to remote hosts in the next step.
git clean -fdxq --exclude='report*.xml'
# scalability with 32cores (and upload to a different BQ table)
tools/run_tests/run_performance_tests.py \
-l c++ java csharp go \
--netperf \
--category scalable \
--bq_result_table performance_test.performance_experiment_32core \
--remote_worker_host grpc-performance-server-32core grpc-performance-client-32core grpc-performance-client2-32core \
--xml_report report_32core.xml \
|| EXIT_CODE=1
# prevent pushing leftover build files to remote hosts in the next step.
git clean -fdxq --exclude='report*.xml'
# selected scenarios on Windows
tools/run_tests/run_performance_tests.py \
-l csharp \
--category scalable \
--bq_result_table performance_test.performance_experiment_windows \
--remote_worker_host grpc-performance-windows1 grpc-performance-windows2 \
--xml_report report_windows.xml \
|| EXIT_CODE=1
exit $EXIT_CODE

@ -1,58 +0,0 @@
#!/usr/bin/env bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# A frozen version of run_full_performance.sh that runs full performance test
# suite for the latest released stable version of gRPC.
set -ex
# Enter the gRPC repo root
cd $(dirname $0)/../..
# run 8core client vs 8core server
tools/run_tests/run_performance_tests.py \
-l c++ csharp node ruby java python go node_express \
--netperf \
--category scalable \
--bq_result_table performance_released.performance_experiment \
--remote_worker_host grpc-performance-server-8core grpc-performance-client-8core grpc-performance-client2-8core \
--xml_report report_8core.xml \
|| EXIT_CODE=1
# prevent pushing leftover build files to remote hosts in the next step.
git clean -fdxq --exclude='report*.xml'
# scalability with 32cores (and upload to a different BQ table)
tools/run_tests/run_performance_tests.py \
-l c++ java csharp go \
--netperf \
--category scalable \
--bq_result_table performance_released.performance_experiment_32core \
--remote_worker_host grpc-performance-server-32core grpc-performance-client-32core grpc-performance-client2-32core \
--xml_report report_32core.xml \
|| EXIT_CODE=1
# prevent pushing leftover build files to remote hosts in the next step.
git clean -fdxq --exclude='report*.xml'
# selected scenarios on Windows
tools/run_tests/run_performance_tests.py \
-l csharp \
--category scalable \
--bq_result_table performance_released.performance_experiment_windows \
--remote_worker_host grpc-performance-windows1 grpc-performance-windows2 \
--xml_report report_windows.xml \
|| EXIT_CODE=1
exit $EXIT_CODE

@ -1,33 +0,0 @@
#!/bin/bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Builds and runs a fuzzer (specified by the first command line argument)
set -ex
export RUN_COMMAND="tools/fuzzer/build_and_run_fuzzer.sh $1"
export DOCKER_RUN_SCRIPT=tools/run_tests/dockerize/docker_run.sh
export DOCKERFILE_DIR=tools/dockerfile/test/fuzzer
export OUTPUT_DIR=fuzzer_output
runtime=${runtime:-3600}
jobs=${jobs:-3}
tools/run_tests/dockerize/build_and_run_docker.sh \
-e RUN_COMMAND="$RUN_COMMAND" \
-e OUTPUT_DIR="$OUTPUT_DIR" \
-e config="$config" \
-e runtime="$runtime" \
-e jobs="$jobs"

@ -1,24 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs interop test suite.
set -ex
export LANG=en_US.UTF-8
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop --http2_server_interop -t -j 12 $@ || true

@ -1,24 +0,0 @@
#!/usr/bin/env bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs interop test suite.
set -ex
export LANG=en_US.UTF-8
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/run_interop_tests.py -l objc -s all --use_docker -t -j 1 $@ || true

@ -1,55 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and triggers a test run based on
# env variable settings.
#
# Setting up rvm environment BEFORE we set -ex.
[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
# To prevent cygwin bash complaining about empty lines ending with \r
# we set the igncr option. The option doesn't exist on Linux, so we fallback
# to just 'set -ex' there.
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
if [ "$platform" == "linux" ]
then
PLATFORM_SPECIFIC_ARGS="--use_docker --measure_cpu_costs"
elif [ "$platform" == "freebsd" ]
then
export MAKE=gmake
fi
unset platform # variable named 'platform' breaks the windows build
python tools/run_tests/run_tests.py \
$PLATFORM_SPECIFIC_ARGS \
-t \
-l $language \
-c $config \
-x report.xml \
-j 2 \
$@ || TESTS_FAILED="true"
if [ ! -e reports/index.html ]
then
mkdir -p reports
echo 'No reports generated.' > reports/index.html
fi
if [ "$TESTS_FAILED" != "" ]
then
exit 1
fi

@ -1,27 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and triggers a test run, bypassing
# all args to the test script.
#
# Setting up rvm environment BEFORE we set -ex.
[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
# To prevent cygwin bash complaining about empty lines ending with \r
# we set the igncr option. The option doesn't exist on Linux, so we fallback
# to just 'set -ex' there.
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
python tools/run_tests/run_tests_matrix.py $@

@ -1,21 +0,0 @@
#!/usr/bin/env bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and counts the number of lines in the
# project.
set -ex
cd $(dirname $0)/../..
tools/line_count/collect-now.sh

@ -1,26 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs a diff on the microbenchmarks
set -ex
# List of benchmarks that provide good signal for analyzing performance changes in pull requests
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/start_port_server.py
tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN

@ -1,50 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs portability tests based on
# env variable setting.
#
# Setting up rvm environment BEFORE we set -ex.
[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
# To prevent cygwin bash complaining about empty lines ending with \r
# we set the igncr option. The option doesn't exist on Linux, so we fallback
# to just 'set -ex' there.
# NOTE: No empty lines should appear in this file before igncr is set!
set -ex -o igncr || set -ex
echo "building $scenario"
# If scenario has _bo suffix, add --build_only flag.
# Short suffix name had to been chosen due to path length limit on Windows.
if [ "$scenario" != "${scenario%_bo}" ]
then
scenario="${scenario%_bo}"
BUILD_ONLY_MAYBE="--build_only"
fi
parts=($(echo $scenario | tr '_' ' ')) # split scenario into parts
curr_platform=${parts[0]} # variable named 'platform' breaks the windows build
curr_arch=${parts[1]}
curr_compiler=${parts[2]}
config='dbg'
if [ "$curr_platform" == "linux" ]
then
USE_DOCKER_MAYBE="--use_docker"
fi
python tools/run_tests/run_tests.py $USE_DOCKER_MAYBE $BUILD_ONLY_MAYBE -t -l $language -c $config --arch ${curr_arch} --compiler ${curr_compiler} -x report.xml -j 3 $@

@ -1,23 +0,0 @@
#!/usr/bin/env bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs a diff on the qps drivers
set -ex
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/start_port_server.py
tools/profiling/qps/qps_diff.py -d origin/$ghprbTargetBranch

@ -1,34 +0,0 @@
#!/usr/bin/env bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs full performance test suite.
set -ex
SERVER_HOST=${1:-grpc-performance-server-32core}
CLIENT_HOST1=${2:-grpc-performance-client-32core}
CLIENT_HOST2=${3:-grpc-performance-client2-32core}
# Enter the gRPC repo root
cd $(dirname $0)/../..
# scalability with 32cores (and upload to a different BQ table)
tools/run_tests/run_performance_tests.py \
-l c++ \
--category sweep \
--bq_result_table performance_test.performance_experiment_32core \
--remote_worker_host ${SERVER_HOST} ${CLIENT_HOST1} ${CLIENT_HOST2} \
--perf_args "record -F 97 --call-graph dwarf" \
|| EXIT_CODE=1
exit $EXIT_CODE

@ -1,23 +0,0 @@
#!/usr/bin/env bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is invoked by Jenkins and runs a diff on bm_fullstack_trickle
set -ex
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/start_port_server.py
tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls_per_iteration cli_stream_stalls_per_iteration svr_transport_stalls_per_iteration svr_stream_stalls_per_iteration --no-counters --pr_comment_name trickle
Loading…
Cancel
Save