From 67d7cdc45ab63d4ee0901e793e895dab238fb551 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 May 2018 14:25:18 -0700 Subject: [PATCH 1/5] Support building sanitizers with Clang5+ --- Makefile | 10 +++++----- build.yaml | 13 +++++++------ tools/run_tests/dockerize/build_and_run_docker.sh | 1 + .../dockerize/build_docker_and_run_tests.sh | 1 + tools/run_tests/dockerize/build_interop_image.sh | 1 + 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 1d1597e3265..97eec226c43 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ CC_asan-trace-cmp = clang CXX_asan-trace-cmp = clang++ LD_asan-trace-cmp = clang++ LDXX_asan-trace-cmp = clang++ -CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-trace-cmp = -fsanitize=address VALID_CONFIG_dbg = 1 @@ -103,7 +103,7 @@ CC_asan = clang CXX_asan = clang++ LD_asan = clang++ LDXX_asan = clang++ -CPPFLAGS_asan = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_asan = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan = -fsanitize=address VALID_CONFIG_msan = 1 @@ -112,7 +112,7 @@ CC_msan = clang CXX_msan = clang++ LD_msan = clang++ LDXX_msan = clang++ -CPPFLAGS_msan = -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_msan = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) DEFINES_msan = NDEBUG @@ -139,7 +139,7 @@ CC_asan-noleaks = clang CXX_asan-noleaks = clang++ LD_asan-noleaks = clang++ LDXX_asan-noleaks = clang++ -CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address VALID_CONFIG_noexcept = 1 @@ -157,7 +157,7 @@ CC_ubsan = clang CXX_ubsan = clang++ LD_ubsan = clang++ LDXX_ubsan = clang++ -CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs +CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow DEFINES_ubsan = NDEBUG GRPC_UBSAN diff --git a/build.yaml b/build.yaml index 6b3955f6a55..16c90fbd4c1 100644 --- a/build.yaml +++ b/build.yaml @@ -5470,7 +5470,7 @@ vspackages: configs: asan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ LD: clang++ @@ -5482,7 +5482,7 @@ configs: LSAN_OPTIONS: suppressions=tools/lsan_suppressions.txt:report_objects=1 asan-noleaks: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ LD: clang++ @@ -5493,8 +5493,9 @@ configs: ASAN_OPTIONS: detect_leaks=0:color=always asan-trace-cmp: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address - -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize-coverage=trace-cmp + -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument + -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ LD: clang++ LDFLAGS: -fsanitize=address @@ -5539,7 +5540,7 @@ configs: valgrind: --tool=memcheck --leak-check=full msan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ @@ -5579,7 +5580,7 @@ configs: TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 ubsan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs CXX: clang++ DEFINES: NDEBUG GRPC_UBSAN diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 3ea3ca3c0bc..4ef74085f94 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -56,6 +56,7 @@ CONTAINER_NAME="build_and_run_docker_$(uuidgen)" # shellcheck disable=SC2086 docker run \ "$@" \ + --cap-add SYS_PTRACE \ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \ diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index 21eccbaac9e..614049cae53 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -53,6 +53,7 @@ DOCKER_EXIT_CODE=0 # TODO: silence complaint about $TTY_FLAG expansion in some other way # shellcheck disable=SC2086 docker run \ + --cap-add SYS_PTRACE \ -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \ -e "config=$config" \ -e "arch=$arch" \ diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh index 2bef4a33102..fcfcdeb4e41 100755 --- a/tools/run_tests/dockerize/build_interop_image.sh +++ b/tools/run_tests/dockerize/build_interop_image.sh @@ -104,6 +104,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" # Same for $TTY_FLAG # shellcheck disable=SC2086 (docker run \ + --cap-add SYS_PTRACE \ -e CCACHE_DIR=/tmp/ccache \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e THIS_IS_REALLY_NEEDED_ONCE_AGAIN='For issue 4835. See https://github.com/docker/docker/issues/14203 for why docker is awful' \ From 528775db084d5561f6bb4bc4c4fa69bd629dfa02 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 May 2018 14:55:40 -0700 Subject: [PATCH 2/5] Add Jessie Dockerfile using RBE's base --- .../test/cxx_jessie_x64/Dockerfile.template | 2 +- .../Dockerfile.template | 31 +++++++ .../dockerfile/test/cxx_jessie_x64/Dockerfile | 2 +- .../test/cxx_sanitizers_jessie_x64/Dockerfile | 80 +++++++++++++++++++ tools/run_tests/run_tests.py | 13 ++- tools/run_tests/run_tests_matrix.py | 8 +- 6 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template create mode 100644 tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile diff --git a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template index 9d559dfafad..81272319083 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template @@ -29,4 +29,4 @@ RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean # Define the default command. - CMD ["bash"] + CMD ["bash"] \ No newline at end of file diff --git a/templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template new file mode 100644 index 00000000000..6070330e919 --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template @@ -0,0 +1,31 @@ +%YAML 1.2 +--- | + # Copyright 2018 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # This is the base Docker image we use for running tests on RBE + FROM gcr.io/cloud-marketplace/google/rbe-debian8@sha256:1ede2a929b44d629ec5abe86eee6d7ffea1d5a4d247489a8867d46cfde3e38bd + + <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../python_deps.include"/> + #================= + # C++ dependencies (purposely excluding Clang because it's part of the base image) + RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev && apt-get clean + + # Link llvm-symbolizer to where our test scripts expect to find it + RUN ln -s /usr/local/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + + # Define the default command. + CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index 4dda0f0b9e3..8a20d92d97e 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -121,4 +121,4 @@ RUN cd /tmp && wget http://dist.libuv.org/dist/v1.9.1/libuv-v1.9.1.tar.gz RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean # Define the default command. -CMD ["bash"] +CMD ["bash"] \ No newline at end of file diff --git a/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile new file mode 100644 index 00000000000..f60f67665dc --- /dev/null +++ b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile @@ -0,0 +1,80 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the base Docker image we use for running tests on RBE +FROM gcr.io/cloud-marketplace/google/rbe-debian8@sha256:1ede2a929b44d629ec5abe86eee6d7ffea1d5a4d247489a8867d46cfde3e38bd + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + dnsutils \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install --upgrade pip==10.0.1 +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 + +#================= +# C++ dependencies (purposely excluding Clang because it's part of the base image) +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev && apt-get clean + +# Link llvm-symbolizer to where our test scripts expect to find it +RUN ln -s /usr/local/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 5aabf9c161b..ca229b2b833 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -531,6 +531,11 @@ class CLanguage(object): elif compiler == 'clang3.7': return ('ubuntu1604', self._clang_make_options(version_suffix='-3.7')) + elif compiler == 'clang7.0': + # clang++-7.0 alias doesn't exist and there are no other clang versions + # installed. + return ('sanitizers_jessie', + self._clang_make_options()) else: raise Exception('Compiler %s not supported.' % compiler) @@ -1343,10 +1348,10 @@ argp.add_argument( '--compiler', choices=[ 'default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc7.2', - 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'python2.7', - 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', - 'all_the_cpythons', 'electron1.3', 'electron1.6', 'coreclr', 'cmake', - 'cmake_vs2015', 'cmake_vs2017' + 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'clang7.0', + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', + 'python_alpine', 'all_the_cpythons', 'electron1.3', 'electron1.6', + 'coreclr', 'cmake', 'cmake_vs2015', 'cmake_vs2017' ], default='default', help= diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 64d80ab721d..0af9e0cbbe1 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -245,6 +245,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): languages=['c'], configs=['msan', 'asan', 'tsan', 'ubsan'], platforms=['linux'], + arch='x64', + compiler='clang7.0', labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, @@ -253,6 +255,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): languages=['c++'], configs=['asan'], platforms=['linux'], + arch='x64', + compiler='clang7.0', labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, @@ -261,6 +265,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): languages=['c++'], configs=['tsan'], platforms=['linux'], + arch='x64', + compiler='clang7.0', labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, @@ -286,7 +292,7 @@ def _create_portability_test_jobs(extra_args=[], # portability C and C++ on x64 for compiler in [ 'gcc4.8', 'gcc5.3', 'gcc7.2', 'gcc_musl', 'clang3.5', 'clang3.6', - 'clang3.7' + 'clang3.7', 'clang7.0' ]: test_jobs += _generate_jobs( languages=['c', 'c++'], From 048bb0713b4952ba6c9340cc944458ce99715fbc Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 May 2018 19:27:21 -0700 Subject: [PATCH 3/5] Remove installed from source Clang in standard Jessie Dockerfile --- .../test/cxx_jessie_x64/Dockerfile.template | 2 - .../dockerfile/test/cxx_jessie_x64/Dockerfile | 38 ------------------- 2 files changed, 40 deletions(-) diff --git a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template index 81272319083..6b00c862f9c 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template @@ -20,8 +20,6 @@ <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> - <%include file="../../cmake_jessie_backports.include"/> - <%include file="../../clang_update.include"/> <%include file="../../run_tests_addons.include"/> <%include file="../../libuv_install.include"/> diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index 8a20d92d97e..ad1fbec5dfa 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -72,44 +72,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean -#================= -# Use cmake 3.6 from jessie-backports -# should only be used for images based on debian jessie. - -RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list -RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean - -#================= -# Update clang to a version with improved tsan and fuzzing capabilities - -RUN git clone -n -b release_38 http://llvm.org/git/llvm.git && \ - cd llvm && git checkout ad57503 && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/clang.git && \ - cd clang && git checkout ad2c56e && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/compiler-rt.git && \ - cd compiler-rt && git checkout 3176922 && cd .. -RUN git clone -n -b release_38 \ - http://llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && \ - git checkout c288525 && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/libcxx.git && \ - cd libcxx && git checkout fda3549 && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/libcxxabi.git && \ - cd libcxxabi && git checkout 8d4e51d && cd .. - -RUN mv clang llvm/tools -RUN mv compiler-rt llvm/projects -RUN mv clang-tools-extra llvm/tools/clang/tools -RUN mv libcxx llvm/projects -RUN mv libcxxabi llvm/projects - -RUN mkdir llvm-build -RUN cd llvm-build && cmake \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_INSTALL_PREFIX:STRING=/usr \ - -DLLVM_TARGETS_TO_BUILD:STRING=X86 \ - ../llvm -RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build - RUN mkdir /var/local/jenkins From cacc2ccbf0d37f57efdbc8f81ae8620d4a78366c Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 22 May 2018 11:07:35 -0700 Subject: [PATCH 4/5] Readd cmake backport to Jessie --- .../dockerfile/test/cxx_jessie_x64/Dockerfile.template | 3 ++- .../dockerfile/test/cxx_jessie_x86/Dockerfile.template | 1 + tools/dockerfile/test/cxx_jessie_x64/Dockerfile | 9 ++++++++- tools/dockerfile/test/cxx_jessie_x86/Dockerfile | 7 +++++++ tools/run_tests/run_tests.py | 3 +-- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template index 6b00c862f9c..01e22cce4c4 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template @@ -20,6 +20,7 @@ <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> + <%include file="../../cmake_jessie_backports.include"/> <%include file="../../run_tests_addons.include"/> <%include file="../../libuv_install.include"/> @@ -27,4 +28,4 @@ RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean # Define the default command. - CMD ["bash"] \ No newline at end of file + CMD ["bash"] diff --git a/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template index 8011448b69e..b9cd1e98285 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template @@ -21,6 +21,7 @@ <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../run_tests_addons.include"/> + <%include file="../../cmake_jessie_backports.include"/> # Install gcc-4.8 and other relevant items RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index ad1fbec5dfa..c4f959fc1dc 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -72,6 +72,13 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean +#================= +# Use cmake 3.6 from jessie-backports +# should only be used for images based on debian jessie. + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + RUN mkdir /var/local/jenkins @@ -83,4 +90,4 @@ RUN cd /tmp && wget http://dist.libuv.org/dist/v1.9.1/libuv-v1.9.1.tar.gz RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean # Define the default command. -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index 84d811dbba1..d07ea9a9b0a 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -75,6 +75,13 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c RUN mkdir /var/local/jenkins +#================= +# Use cmake 3.6 from jessie-backports +# should only be used for images based on debian jessie. + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + # Install gcc-4.8 and other relevant items RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ca229b2b833..5d89cacc549 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -534,8 +534,7 @@ class CLanguage(object): elif compiler == 'clang7.0': # clang++-7.0 alias doesn't exist and there are no other clang versions # installed. - return ('sanitizers_jessie', - self._clang_make_options()) + return ('sanitizers_jessie', self._clang_make_options()) else: raise Exception('Compiler %s not supported.' % compiler) From 6578c34b73b89e3d00176a73b498d127d11e4795 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 22 May 2018 18:15:48 -0700 Subject: [PATCH 5/5] Force UBSan to use clang++ as ld --- tools/run_tests/run_tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 5d89cacc549..ea4c7c3c655 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -490,6 +490,14 @@ class CLanguage(object): return 'Makefile' def _clang_make_options(self, version_suffix=''): + if self.args.config == 'ubsan': + return [ + 'CC=clang%s' % version_suffix, + 'CXX=clang++%s' % version_suffix, + 'LD=clang++%s' % version_suffix, + 'LDXX=clang++%s' % version_suffix + ] + return [ 'CC=clang%s' % version_suffix, 'CXX=clang++%s' % version_suffix,