|
|
|
#!/bin/bash
|
|
|
|
# Copyright 2015, Google Inc.
|
|
|
|
# All rights reserved.
|
|
|
|
#
|
|
|
|
# Redistribution and use in source and binary forms, with or without
|
|
|
|
# modification, are permitted provided that the following conditions are
|
|
|
|
# met:
|
|
|
|
#
|
|
|
|
# * Redistributions of source code must retain the above copyright
|
|
|
|
# notice, this list of conditions and the following disclaimer.
|
|
|
|
# * Redistributions in binary form must reproduce the above
|
|
|
|
# copyright notice, this list of conditions and the following disclaimer
|
|
|
|
# in the documentation and/or other materials provided with the
|
|
|
|
# distribution.
|
|
|
|
# * Neither the name of Google Inc. nor the names of its
|
|
|
|
# contributors may be used to endorse or promote products derived from
|
|
|
|
# this software without specific prior written permission.
|
|
|
|
#
|
|
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
|
|
# Initializes a fresh GCE VM to become a jenkins linux performance worker.
|
|
|
|
# You shouldn't run this script on your own,
|
|
|
|
# use create_linux_performance_worker.sh instead.
|
|
|
|
|
|
|
|
set -ex
|
|
|
|
|
|
|
|
sudo apt-get update
|
|
|
|
|
|
|
|
# Install Java 8 JDK (to build gRPC Java)
|
|
|
|
sudo apt-get install -y openjdk-8-jdk
|
|
|
|
sudo apt-get install -y unzip lsof
|
|
|
|
|
|
|
|
sudo apt-get install -y \
|
|
|
|
autoconf \
|
|
|
|
autotools-dev \
|
|
|
|
build-essential \
|
|
|
|
bzip2 \
|
|
|
|
ccache \
|
|
|
|
curl \
|
|
|
|
gcc \
|
|
|
|
gcc-multilib \
|
|
|
|
git \
|
|
|
|
gyp \
|
|
|
|
lcov \
|
|
|
|
libc6 \
|
|
|
|
libc6-dbg \
|
|
|
|
libc6-dev \
|
|
|
|
libcurl4-openssl-dev \
|
|
|
|
libgtest-dev \
|
|
|
|
libreadline-dev \
|
|
|
|
libssl-dev \
|
|
|
|
libtool \
|
|
|
|
make \
|
|
|
|
strace \
|
|
|
|
pypy \
|
|
|
|
python-dev \
|
|
|
|
python-pip \
|
|
|
|
python-setuptools \
|
|
|
|
python-yaml \
|
|
|
|
python3-dev \
|
|
|
|
python3-pip \
|
|
|
|
python3-setuptools \
|
|
|
|
python3-yaml \
|
|
|
|
telnet \
|
|
|
|
unzip \
|
|
|
|
wget \
|
|
|
|
zip \
|
|
|
|
zlib1g-dev
|
|
|
|
|
|
|
|
# perftools
|
|
|
|
sudo apt-get install -y google-perftools libgoogle-perftools-dev
|
|
|
|
|
|
|
|
# netperf
|
|
|
|
sudo apt-get install -y netperf
|
|
|
|
|
|
|
|
# C++ dependencies
|
|
|
|
sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang
|
|
|
|
|
|
|
|
# Python dependencies
|
|
|
|
sudo pip install tabulate
|
|
|
|
sudo pip install google-api-python-client
|
|
|
|
sudo pip install virtualenv
|
|
|
|
|
|
|
|
# Building gRPC Python depends on python3.4 being installed, but python3.4
|
|
|
|
# is not available on Ubuntu 16.10, so install from source
|
|
|
|
curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz
|
|
|
|
tar xzvf Python-3.4.6.tgz
|
|
|
|
cd Python-3.4.6
|
|
|
|
./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib"
|
|
|
|
sudo make altinstall
|
|
|
|
cd ..
|
|
|
|
rm Python-3.4.6.tgz
|
|
|
|
|
|
|
|
curl -O https://bootstrap.pypa.io/get-pip.py
|
|
|
|
sudo pypy get-pip.py
|
|
|
|
sudo pypy -m pip install tabulate
|
|
|
|
sudo pip install google-api-python-client
|
|
|
|
|
|
|
|
# Node dependencies (nvm has to be installed under user jenkins)
|
|
|
|
touch .profile
|
|
|
|
curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash
|
|
|
|
source ~/.nvm/nvm.sh
|
|
|
|
nvm install 0.12 && npm config set cache /tmp/npm-cache
|
|
|
|
nvm install 4 && npm config set cache /tmp/npm-cache
|
|
|
|
nvm install 5 && npm config set cache /tmp/npm-cache
|
|
|
|
nvm alias default 4
|
|
|
|
|
|
|
|
# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives)
|
|
|
|
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
|
|
|
echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
|
|
|
|
sudo apt-get update
|
|
|
|
sudo apt-get install -y mono-devel nuget
|
|
|
|
|
|
|
|
# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu)
|
|
|
|
sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list'
|
|
|
|
sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893
|
|
|
|
sudo apt-get update
|
|
|
|
sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155
|
|
|
|
sudo apt-get install -y dotnet-dev-1.0.1
|
|
|
|
|
|
|
|
# Ruby dependencies
|
|
|
|
git clone https://github.com/rbenv/rbenv.git ~/.rbenv
|
|
|
|
export PATH="$HOME/.rbenv/bin:$PATH"
|
|
|
|
eval "$(rbenv init -)"
|
|
|
|
|
|
|
|
git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
|
|
|
|
export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"
|
|
|
|
|
|
|
|
rbenv install 2.4.0
|
|
|
|
rbenv global 2.4.0
|
|
|
|
ruby -v
|
|
|
|
|
|
|
|
# Install bundler (prerequisite for gRPC Ruby)
|
|
|
|
gem install bundler
|
|
|
|
|
|
|
|
# Java dependencies - nothing as we already have Java JDK 8
|
|
|
|
|
|
|
|
# Go dependencies
|
|
|
|
# Currently, the golang package available via apt-get doesn't have the latest go.
|
|
|
|
# Significant performance improvements with grpc-go have been observed after
|
|
|
|
# upgrading from go 1.5 to a later version, so a later go version is preferred.
|
|
|
|
# Following go install instructions from https://golang.org/doc/install
|
|
|
|
GO_VERSION=1.8
|
|
|
|
OS=linux
|
|
|
|
ARCH=amd64
|
|
|
|
curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz
|
|
|
|
sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
|
|
|
|
# Put go on the PATH, keep the usual installation dir
|
|
|
|
sudo ln -s /usr/local/go/bin/go /usr/bin/go
|
|
|
|
rm go$GO_VERSION.$OS-$ARCH.tar.gz
|
|
|
|
|
|
|
|
# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version)
|
|
|
|
sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`
|
|
|
|
# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar
|
|
|
|
echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid
|
|
|
|
# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol
|
|
|
|
echo 0 | sudo tee /proc/sys/kernel/kptr_restrict
|
|
|
|
|
|
|
|
# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in
|
|
|
|
# order to not lose perf events or time out
|
|
|
|
echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb
|
|
|
|
|
|
|
|
# Fetch scripts to generate flame graphs from perf data collected
|
|
|
|
# on benchmarks
|
|
|
|
git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph
|
|
|
|
|
|
|
|
# Install scipy and numpy for benchmarking scripts
|
|
|
|
sudo apt-get install -y python-scipy python-numpy
|
|
|
|
|
|
|
|
# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@
|
|
|
|
# This needs to happen as the last step to prevent Jenkins master from connecting
|
|
|
|
# to a machine that hasn't been properly setup yet.
|
|
|
|
cat jenkins_master.pub | sudo tee --append ~jenkins/.ssh/authorized_keys
|
|
|
|
|
|
|
|
# Restart for VM to pick up kernel update
|
|
|
|
echo 'Successfully initialized the linux worker, going for reboot in 10 seconds'
|
|
|
|
sleep 10
|
|
|
|
sudo reboot
|