Merge pull request #13538 from matt-kwong/kokoro-multi-vm

Add multi-lang performance benchmarking jobs to Kokoro

.gitignore

@@ -56,6 +56,7 @@ Gemfile.lock
 
 # Temporary test reports
 report.xml
+*/sponge_log.xml
 latency_trace.txt
 latency_trace.*.txt
 

tools/gce/linux_kokoro_performance_worker_init.sh

@@ -114,6 +114,19 @@ sudo apt-get update
 sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155
 sudo apt-get install -y dotnet-dev-1.0.1
 
+# C# 1.0.4 SDK
+curl -O https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz
+sudo mkdir -p /opt/dotnet
+sudo tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C /opt/dotnet
+sudo ln -s /opt/dotnet/dotnet /usr/local/bin
+
+# C# .NET dependencies
+wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu52_52.1-8ubuntu0.2_amd64.deb
+sudo dpkg -i libicu52_52.1-8ubuntu0.2_amd64.deb
+wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.3_amd64.deb
+sudo dpkg -i libicu55_55.1-7ubuntu0.3_amd64.deb
+sudo apt-get update && sudo apt-get install -y libicu55
+
 # Ruby dependencies
 gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
 curl -sSL https://get.rvm.io | bash -s stable --ruby

tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Source this rc script to prepare the environment for linux perf builds
+
+# Need to increase open files limit and size for perf test
+ulimit -n 32768
+ulimit -c unlimited
+
+# Download non-core gRPC repos
+git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go
+git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java
+git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node
+
+sudo pip install tabulate
+
+# Set up Ruby
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+gem list bundler
+gem install bundler --no-ri --no-rdoc
+
+# Allow SSH to Kokoro performance workers without explicit key verification
+gsutil cp gs://grpc-testing-secrets/grpc_kokoro_performance_ssh_keys/id_rsa ~/.ssh
+echo -e 'Host grpc-kokoro-performance*\n\tStrictHostKeyChecking no' >> ~/.ssh/config
+chmod 600 ~/.ssh/id_rsa ~/.ssh/config
+
+git submodule update --init

tools/internal_ci/linux/grpc_full_performance_master.cfg

@@ -0,0 +1,25 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_master.sh"
+timeout_mins: 600
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "**/perf_reports/**"
+  }
+}

tools/internal_ci/linux/grpc_full_performance_master.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc
+
+# run 8core client vs 8core server
+tools/run_tests/run_performance_tests.py \
+    -l c++ csharp ruby java python go php7 php7_protobuf_c \
+    --netperf \
+    --category scalable \
+    --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \
+    -u kbuilder \
+    --bq_result_table performance_test.kokoro_performance_experiment \
+    --xml_report reports/8core/sponge_log.xml \
+    || EXIT_CODE=1
+
+# prevent pushing leftover build files to remote hosts in the next step.
+git clean -fdxq -e reports
+
+# scalability with 32cores (and upload to a different BQ table)
+tools/run_tests/run_performance_tests.py \
+    -l c++ java csharp go \
+    --netperf \
+    --category scalable \
+    --remote_worker_host grpc-kokoro-performance-server-32core grpc-kokoro-performance-client-32core grpc-kokoro-performance-client2-32core \
+    -u kbuilder \
+    --bq_result_table performance_test.kokoro_performance_experiment_32core \
+    --xml_report reports/32core/sponge_log.xml \
+    || EXIT_CODE=1
+
+# prevent pushing leftover build files to remote hosts in the next step.
+git clean -fdxq -e reports
+
+# selected scenarios on Windows
+tools/run_tests/run_performance_tests.py \
+    -l csharp \
+    --category scalable \
+    --remote_worker_host grpc-kokoro-performance-windows1 grpc-kokoro-performance-windows2 \
+    --bq_result_table performance_test.kokoro_performance_experiment_windows \
+    --xml_report reports/windows/sponge_log.xml \
+    || EXIT_CODE=1
+
+exit $EXIT_CODE

tools/run_tests/run_performance_tests.py

@@ -196,7 +196,7 @@ def archive_repo(languages):
 
 def prepare_remote_hosts(hosts, prepare_local=False):
   """Prepares remote hosts (and maybe prepare localhost as well)."""
-  prepare_timeout = 5*60
+  prepare_timeout = 10*60
   prepare_jobs = []
   for host in hosts:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
@@ -229,6 +229,8 @@ def prepare_remote_hosts(hosts, prepare_local=False):
 def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False):
   """Builds performance worker on remote hosts (and maybe also locally)."""
   build_timeout = 15*60
+  # Kokoro VMs (which are local only) do not have caching, so they need more time to build
+  local_build_timeout = 30*60
   build_jobs = []
   for host in hosts:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
@@ -245,7 +247,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui
             cmdline=['tools/run_tests/performance/build_performance.sh'] + languages,
             shortname='local_build',
             environ = {'CONFIG': 'opt'},
-            timeout_seconds=build_timeout))
+            timeout_seconds=local_build_timeout))
   jobset.message('START', 'Building.', do_newline=True)
   num_failures, _ = jobset.run(
       build_jobs, newline_on_success=True, maxjobs=10)
@@ -483,9 +485,15 @@ def main():
                           'generating flamegraphs (e.g., "--perf_args=stat ...")'))
   argp.add_argument('-f', '--flame_graph_reports', default='perf_reports', type=str,
                     help='Name of directory to output flame graph profiles to, if any are created.')
+  argp.add_argument('-u', '--remote_host_username', default='', type=str,
+                    help='Use a username that isn\'t "Jenkins" to SSH into remote workers.')
 
   args = argp.parse_args()
 
+  global _REMOTE_HOST_USERNAME
+  if args.remote_host_username:
+    _REMOTE_HOST_USERNAME = args.remote_host_username
+
   languages = set(scenario_config.LANGUAGES[l]
                   for l in itertools.chain.from_iterable(
                         six.iterkeys(scenario_config.LANGUAGES) if x == 'all'