Merge pull request #17245 from jtattermusch/coverage_split_up

Split up test coverage runs by language

templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template

@@ -1,41 +0,0 @@
-%YAML 1.2
---- |
-  # Copyright 2016 gRPC authors.
-  #
-  # Licensed under the Apache License, Version 2.0 (the "License");
-  # you may not use this file except in compliance with the License.
-  # You may obtain a copy of the License at
-  #
-  #     http://www.apache.org/licenses/LICENSE-2.0
-  #
-  # Unless required by applicable law or agreed to in writing, software
-  # distributed under the License is distributed on an "AS IS" BASIS,
-  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  # See the License for the specific language governing permissions and
-  # limitations under the License.
-  
-  FROM debian:jessie
-  
-  <%include file="../../apt_get_basic.include"/>
-  <%include file="../../gcp_api_libraries.include"/>
-  <%include file="../../csharp_deps.include"/>
-  <%include file="../../csharp_dotnetcli_deps.include"/>
-  <%include file="../../cxx_deps.include"/>
-  <%include file="../../node_deps.include"/>
-  <%include file="../../php_deps.include"/>
-  <%include file="../../ruby_deps.include"/>
-  <%include file="../../python_deps.include"/>
-  # Install pip and virtualenv for Python 3.4
-  RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4
-  RUN python3.4 -m pip install virtualenv
-  
-  # Install coverage for Python test coverage reporting
-  RUN pip install coverage
-  ENV PATH ~/.local/bin:$PATH
-
-  # Install Mako to generate files in grpc/grpc-node
-  RUN pip install Mako
-
-  <%include file="../../run_tests_addons.include"/>
-  # Define the default command.
-  CMD ["bash"]

templates/tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile.template

@@ -18,3 +18,6 @@
 
   RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
   RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+
+  # for Python test coverage reporting
+  RUN pip install coverage

tools/dockerfile/test/multilang_jessie_x64/Dockerfile

@@ -1,170 +0,0 @@
-# Copyright 2016 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM debian:jessie
-
-# Install Git and basic packages.
-RUN apt-get update && apt-get install -y \
-  autoconf \
-  autotools-dev \
-  build-essential \
-  bzip2 \
-  ccache \
-  curl \
-  dnsutils \
-  gcc \
-  gcc-multilib \
-  git \
-  golang \
-  gyp \
-  lcov \
-  libc6 \
-  libc6-dbg \
-  libc6-dev \
-  libgtest-dev \
-  libtool \
-  make \
-  perl \
-  strace \
-  python-dev \
-  python-setuptools \
-  python-yaml \
-  telnet \
-  unzip \
-  wget \
-  zip && apt-get clean
-
-#================
-# Build profiling
-RUN apt-get update && apt-get install -y time && apt-get clean
-
-# Google Cloud platform API libraries
-RUN apt-get update && apt-get install -y python-pip && apt-get clean
-RUN pip install --upgrade google-api-python-client oauth2client
-
-#================
-# C# dependencies
-
-# Update to a newer version of mono
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list
-RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list
-RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list
-
-# Install dependencies
-RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \
-    mono-devel \
-    ca-certificates-mono \
-    nuget \
-    && apt-get clean
-
-RUN nuget update -self
-
-#=================
-# Use cmake 3.6 from jessie-backports
-# needed to build grpc_csharp_ext with cmake
-
-RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list
-RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean
-
-# Install dotnet SDK based on https://www.microsoft.com/net/core#debian
-RUN apt-get update && apt-get install -y curl libunwind8 gettext
-# dotnet-dev-1.0.0-preview2-003131
-RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530
-RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet
-# dotnet-dev-1.0.1
-RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453
-RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet
-RUN ln -s /opt/dotnet/dotnet /usr/local/bin
-
-# Trigger the population of the local package cache
-ENV NUGET_XMLDOC_MODE skip
-RUN mkdir warmup \
-    && cd warmup \
-    && dotnet new \
-    && cd .. \
-    && rm -rf warmup
-
-#=================
-# C++ dependencies
-RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean
-
-#==================
-# Node dependencies
-
-# Install nvm
-RUN touch .profile
-RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash
-# Install all versions of node that we want to test
-RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache"
-RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache"
-RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache"
-RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache"
-RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache"
-RUN /bin/bash -l -c "nvm install 10 && npm config set cache /tmp/npm-cache"
-RUN /bin/bash -l -c "nvm alias default 10"
-#=================
-# PHP dependencies
-
-# Install dependencies
-
-RUN apt-get update && apt-get install -y \
-    git php5 php5-dev phpunit unzip
-
-#==================
-# Ruby dependencies
-
-# Install rvm
-RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-RUN \curl -sSL https://get.rvm.io | bash -s stable
-
-# Install Ruby 2.1
-RUN /bin/bash -l -c "rvm install ruby-2.1"
-RUN /bin/bash -l -c "rvm use --default ruby-2.1"
-RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
-RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
-RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
-RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
-
-#====================
-# Python dependencies
-
-# Install dependencies
-
-RUN apt-get update && apt-get install -y \
-    python-all-dev \
-    python3-all-dev \
-    python-pip
-
-# Install Python packages from PyPI
-RUN pip install --upgrade pip==10.0.1
-RUN pip install virtualenv
-RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0
-
-# Install pip and virtualenv for Python 3.4
-RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4
-RUN python3.4 -m pip install virtualenv
-
-# Install coverage for Python test coverage reporting
-RUN pip install coverage
-ENV PATH ~/.local/bin:$PATH
-
-# Install Mako to generate files in grpc/grpc-node
-RUN pip install Mako
-
-
-RUN mkdir /var/local/jenkins
-
-# Define the default command.
-CMD ["bash"]

tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile

@@ -70,3 +70,6 @@ CMD ["bash"]
 
 RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+
+# for Python test coverage reporting
+RUN pip install coverage

tools/internal_ci/linux/grpc_coverage.sh

@@ -21,12 +21,20 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 python tools/run_tests/run_tests.py \
-  --use_docker                      \
+   -l c c++ -x coverage_cpp/sponge_log.xml \
-  -t                                \
+   --use_docker -t -c gcov -j 2 || FAILED="true"
-  -l all                            \
+
-  -c gcov                           \
+python tools/run_tests/run_tests.py \
-  -x sponge_log.xml                 \
+   -l python -x coverage_python/sponge_log.xml \
-  -j 16 || FAILED="true"
+   --use_docker -t -c gcov -j 2 || FAILED="true"
+
+python tools/run_tests/run_tests.py \
+   -l ruby -x coverage_ruby/sponge_log.xml \
+   --use_docker -t -c gcov -j 2 || FAILED="true"
+
+python tools/run_tests/run_tests.py \
+   -l php -x coverage_php/sponge_log.xml \
+   --use_docker -t -c gcov -j 2 || FAILED="true"
   
 # HTML reports can't be easily displayed in GCS, so create a zip archive
 # and put it under reports directory to get it uploaded as an artifact.

tools/run_tests/dockerize/docker_run_tests.sh

@@ -38,16 +38,8 @@ exit_code=0
 
 $RUN_TESTS_COMMAND || exit_code=$?
 
-cd reports
+# The easiest way to copy all the reports files from inside of
-echo '<html><head></head><body>' > index.html
+# the docker container is to zip them and then copy the zip.
-find . -maxdepth 1 -mindepth 1 -type d | sort | while read d ; do
-  d=${d#*/}
-  n=${d//_/ }
-  echo "<a href='$d/index.html'>$n</a><br />" >> index.html
-done
-echo '</body></html>' >> index.html
-cd ..
-
 zip -r reports.zip reports
 find . -name report.xml -print0 | xargs -0 -r zip reports.zip
 find . -name sponge_log.xml -print0 | xargs -0 -r zip reports.zip

tools/run_tests/helper_scripts/run_lcov.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-# Copyright 2015 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-out=$(readlink -f "${1:-coverage}")
-
-root=$(readlink -f "$(dirname "$0")/../../..")
-shift || true
-tmp=$(mktemp)
-cd "$root"
-tools/run_tests/run_tests.py -c gcov -l c c++ "$@" || true
-lcov --capture --directory . --output-file "$tmp"
-genhtml "$tmp" --output-directory "$out"
-rm "$tmp"
-if which xdg-open > /dev/null
-then
-  xdg-open "file://$out/index.html"
-fi

tools/run_tests/run_tests.py

@@ -1342,9 +1342,9 @@ argp.add_argument(
 argp.add_argument(
     '-l',
     '--language',
-    choices=['all'] + sorted(_LANGUAGES.keys()),
+    choices=sorted(_LANGUAGES.keys()),
     nargs='+',
-    default=['all'])
+    required=True)
 argp.add_argument(
     '-S', '--stop_on_failure', default=False, action='store_const', const=True)
 argp.add_argument(
@@ -1515,17 +1515,7 @@ build_config = run_config.build_config
 if args.travis:
     _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
 
-if 'all' in args.language:
+languages = set(_LANGUAGES[l] for l in args.language)
-    lang_list = list(_LANGUAGES.keys())
-else:
-    lang_list = args.language
-# We don't support code coverage on some languages
-if 'gcov' in args.config:
-    for bad in ['csharp', 'grpc-node', 'objc', 'sanity']:
-        if bad in lang_list:
-            lang_list.remove(bad)
-
-languages = set(_LANGUAGES[l] for l in lang_list)
 for l in languages:
     l.configure(run_config, args)
 
@@ -1537,7 +1527,7 @@ if any(language.make_options() for language in languages):
         )
         sys.exit(1)
     else:
-        # Combining make options is not clean and just happens to work. It allows C/C++ and C# to build
+        # Combining make options is not clean and just happens to work. It allows C & C++ to build
         # together, and is only used under gcov. All other configs should build languages individually.
         language_make_options = list(
             set([