Jan Tattermusch
7 years ago
parent
8ae3285323
commit
607c98d343
4 changed files with 0 additions and 288 deletions
@ -1,50 +0,0 @@ |
||||
#!/bin/bash |
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Creates an interop worker on GCE. |
||||
# IMPORTANT: After this script finishes, there are still some manual |
||||
# steps needed there are hard to automatize. |
||||
# See go/grpc-jenkins-setup for followup instructions. |
||||
|
||||
set -ex |
||||
|
||||
cd "$(dirname "$0")" |
||||
|
||||
CLOUD_PROJECT=grpc-testing |
||||
ZONE=us-east1-a # canary gateway is reachable from this zone |
||||
|
||||
INSTANCE_NAME="${1:-grpc-canary-interop2}" |
||||
|
||||
gcloud compute instances create "$INSTANCE_NAME" \ |
||||
--project="$CLOUD_PROJECT" \ |
||||
--zone "$ZONE" \ |
||||
--machine-type n1-standard-16 \ |
||||
--image ubuntu-15-10 \ |
||||
--boot-disk-size 1000 \ |
||||
--scopes https://www.googleapis.com/auth/xapi.zoo \ |
||||
--tags=allow-ssh |
||||
|
||||
echo 'Created GCE instance, waiting 60 seconds for it to come online.' |
||||
sleep 60 |
||||
|
||||
gcloud compute copy-files \ |
||||
--project="$CLOUD_PROJECT" \ |
||||
--zone "$ZONE" \ |
||||
jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ |
||||
|
||||
gcloud compute ssh \ |
||||
--project="$CLOUD_PROJECT" \ |
||||
--zone "$ZONE" \ |
||||
"$INSTANCE_NAME" --command "./linux_worker_init.sh" |
||||
@ -1,53 +0,0 @@ |
||||
#!/bin/bash |
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Creates a performance worker on GCE. |
||||
# IMPORTANT: After creating the worker, one needs to manually add the pubkey |
||||
# of jenkins@the-machine-where-jenkins-starts-perf-tests |
||||
# to ~/.ssh/authorized_keys so that multi-machine scenarios can work. |
||||
# See tools/run_tests/run_performance_tests.py for details. |
||||
|
||||
set -ex |
||||
|
||||
cd "$(dirname "$0")" |
||||
|
||||
CLOUD_PROJECT=grpc-testing |
||||
ZONE=us-central1-b # this zone allows 32core machines |
||||
|
||||
INSTANCE_NAME="${1:-grpc-performance-server1}" |
||||
MACHINE_TYPE=n1-standard-32 |
||||
|
||||
gcloud compute instances create "$INSTANCE_NAME" \ |
||||
--project="$CLOUD_PROJECT" \ |
||||
--zone "$ZONE" \ |
||||
--machine-type $MACHINE_TYPE \ |
||||
--image-project ubuntu-os-cloud \ |
||||
--image-family ubuntu-1710 \ |
||||
--boot-disk-size 300 \ |
||||
--scopes https://www.googleapis.com/auth/bigquery \ |
||||
--tags=allow-ssh |
||||
|
||||
echo 'Created GCE instance, waiting 60 seconds for it to come online.' |
||||
sleep 60 |
||||
|
||||
gcloud compute copy-files \ |
||||
--project="$CLOUD_PROJECT" \ |
||||
--zone "$ZONE" \ |
||||
jenkins_master.pub linux_performance_worker_init.sh "jenkins@${INSTANCE_NAME}":~ |
||||
|
||||
gcloud compute ssh \ |
||||
--project="$CLOUD_PROJECT" \ |
||||
--zone "$ZONE" \ |
||||
"jenkins@${INSTANCE_NAME}" --command "./linux_performance_worker_init.sh" |
||||
@ -1 +0,0 @@ |
||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master |
||||
@ -1,184 +0,0 @@ |
||||
#!/bin/bash |
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Initializes a fresh GCE VM to become a jenkins linux performance worker. |
||||
# You shouldn't run this script on your own, |
||||
# use create_linux_performance_worker.sh instead. |
||||
|
||||
set -ex |
||||
|
||||
sudo apt-get update |
||||
|
||||
# Install Java 8 JDK (to build gRPC Java) |
||||
sudo apt-get install -y openjdk-8-jdk |
||||
sudo apt-get install -y unzip lsof |
||||
|
||||
sudo apt-get install -y \ |
||||
autoconf \ |
||||
autotools-dev \ |
||||
build-essential \ |
||||
bzip2 \ |
||||
ccache \ |
||||
curl \ |
||||
gcc \ |
||||
gcc-multilib \ |
||||
git \ |
||||
gyp \ |
||||
lcov \ |
||||
libc6 \ |
||||
libc6-dbg \ |
||||
libc6-dev \ |
||||
libcurl4-openssl-dev \ |
||||
libgtest-dev \ |
||||
libreadline-dev \ |
||||
libssl-dev \ |
||||
libtool \ |
||||
make \ |
||||
strace \ |
||||
pypy \ |
||||
python-dev \ |
||||
python-pip \ |
||||
python-setuptools \ |
||||
python-yaml \ |
||||
python3-dev \ |
||||
python3-pip \ |
||||
python3-setuptools \ |
||||
python3-yaml \ |
||||
telnet \ |
||||
unzip \ |
||||
wget \ |
||||
zip \ |
||||
zlib1g-dev |
||||
|
||||
# perftools |
||||
sudo apt-get install -y google-perftools libgoogle-perftools-dev |
||||
|
||||
# netperf |
||||
sudo apt-get install -y netperf |
||||
|
||||
# C++ dependencies |
||||
sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang |
||||
|
||||
# Python dependencies |
||||
sudo pip install --upgrade pip==10.0.1 |
||||
sudo pip install tabulate |
||||
sudo pip install google-api-python-client |
||||
sudo pip install virtualenv |
||||
|
||||
# Building gRPC Python depends on python3.4 being installed, but python3.4 |
||||
# is not available on Ubuntu 16.10, so install from source |
||||
curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz |
||||
tar xzvf Python-3.4.6.tgz |
||||
( |
||||
cd Python-3.4.6 || exit |
||||
./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" |
||||
sudo make altinstall |
||||
) |
||||
rm Python-3.4.6.tgz |
||||
|
||||
curl -O https://bootstrap.pypa.io/get-pip.py |
||||
sudo pypy get-pip.py |
||||
sudo pypy -m pip install tabulate |
||||
sudo pip install google-api-python-client |
||||
|
||||
# Node dependencies (nvm has to be installed under user jenkins) |
||||
touch .profile |
||||
curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash |
||||
# silence shellcheck warning as it cannot follow the `source` path statically: |
||||
# shellcheck disable=SC1090 |
||||
source ~/.nvm/nvm.sh |
||||
nvm install 0.12 && npm config set cache /tmp/npm-cache |
||||
nvm install 4 && npm config set cache /tmp/npm-cache |
||||
nvm install 5 && npm config set cache /tmp/npm-cache |
||||
nvm alias default 4 |
||||
|
||||
# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) |
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
||||
echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list |
||||
sudo apt-get update |
||||
sudo apt-get install -y mono-devel nuget |
||||
|
||||
# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) |
||||
sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' |
||||
sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 |
||||
sudo apt-get update |
||||
sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 |
||||
sudo apt-get install -y dotnet-dev-1.0.1 |
||||
|
||||
# Ruby dependencies |
||||
git clone https://github.com/rbenv/rbenv.git ~/.rbenv |
||||
export PATH="$HOME/.rbenv/bin:$PATH" |
||||
eval "$(rbenv init -)" |
||||
|
||||
git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build |
||||
export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" |
||||
|
||||
rbenv install 2.4.0 |
||||
rbenv global 2.4.0 |
||||
ruby -v |
||||
|
||||
# Install bundler (prerequisite for gRPC Ruby) |
||||
gem install bundler |
||||
|
||||
# PHP dependencies |
||||
sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev |
||||
curl -sS https://getcomposer.org/installer | php |
||||
sudo mv composer.phar /usr/local/bin/composer |
||||
|
||||
# Java dependencies - nothing as we already have Java JDK 8 |
||||
|
||||
# Go dependencies |
||||
# Currently, the golang package available via apt-get doesn't have the latest go. |
||||
# Significant performance improvements with grpc-go have been observed after |
||||
# upgrading from go 1.5 to a later version, so a later go version is preferred. |
||||
# Following go install instructions from https://golang.org/doc/install |
||||
GO_VERSION=1.8 |
||||
OS=linux |
||||
ARCH=amd64 |
||||
curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz |
||||
sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz |
||||
# Put go on the PATH, keep the usual installation dir |
||||
sudo ln -s /usr/local/go/bin/go /usr/bin/go |
||||
rm go$GO_VERSION.$OS-$ARCH.tar.gz |
||||
|
||||
# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) |
||||
sudo apt-get install -y linux-tools-common linux-tools-generic "linux-tools-$(uname -r)" |
||||
# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar |
||||
echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid |
||||
# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol |
||||
echo 0 | sudo tee /proc/sys/kernel/kptr_restrict |
||||
|
||||
# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in |
||||
# order to not lose perf events or time out |
||||
echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb |
||||
|
||||
# Fetch scripts to generate flame graphs from perf data collected |
||||
# on benchmarks |
||||
git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph |
||||
|
||||
# Install scipy and numpy for benchmarking scripts |
||||
sudo apt-get install -y python-scipy python-numpy |
||||
|
||||
# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ |
||||
# This needs to happen as the last step to prevent Jenkins master from connecting |
||||
# to a machine that hasn't been properly setup yet. |
||||
# silence false-positive shellcheck warning ("< redirect does not affect sudo") |
||||
# shellcheck disable=SC2024 |
||||
sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub |
||||
|
||||
# Restart for VM to pick up kernel update |
||||
echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' |
||||
sleep 10 |
||||
sudo reboot |
||||
Loading…
Reference in new issue