From 1cf6b085f2d43649c6d6d475b965e6cea4e4a9a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 25 Jan 2022 20:37:04 +0100 Subject: [PATCH] Enable ccache in more builds (#28665) * upgrade ruby docker images to debian 11 * upgrade C# docker images to debian11 * update sanity dockerimage to debian11 * upgrade cxx interop to debian11 * add ccache to python interop images * enable use of ccache for interop tests * adjust run_tests.py to new docker images * add ccache to rake-compiler-dock docker images * improve prepare_ccache_symlinks_rc * enable use of ccache in rake-compiler-dock docker containers * add ccache support for python_manylinux2014_aarch64 * add ccache support for python_linux_armv7 * deduplicate python3.9 install * ccache for crosscompiled darwin gems is broken * fix bash -l resetting of PATH in grpc_artifact_python_linux_armv7 --- Rakefile | 18 ++++++++ .../grpc_interop_csharp/Dockerfile.template | 4 +- .../Dockerfile.template | 4 +- .../grpc_interop_cxx/Dockerfile.template | 6 ++- .../grpc_interop_python/Dockerfile.template | 3 ++ .../Dockerfile.template | 3 ++ .../grpc_interop_ruby/Dockerfile.template | 5 ++- .../Dockerfile.template | 4 +- .../Dockerfile.template | 5 +-- .../test/sanity/Dockerfile.template | 2 +- .../rake_x64-mingw32/Dockerfile | 14 +++++++ .../rake_x86-linux/Dockerfile | 14 +++++++ .../rake_x86-mingw32/Dockerfile | 14 +++++++ .../rake_x86_64-linux/Dockerfile | 14 +++++++ .../Dockerfile | 30 +++++++++++++- .../Dockerfile | 20 +++++++++ .../grpc_interop_csharp/Dockerfile | 17 +++++++- .../grpc_interop_csharpcoreclr/Dockerfile | 17 +++++++- .../interoptest/grpc_interop_cxx/Dockerfile | 26 ++++++++---- .../grpc_interop_python/Dockerfile | 20 +++++++++ .../grpc_interop_pythonasyncio/Dockerfile | 21 ++++++++++ .../interoptest/grpc_interop_ruby/Dockerfile | 23 ++++++++++- .../Dockerfile | 17 +++++++- .../Dockerfile | 18 ++++++-- tools/dockerfile/test/sanity/Dockerfile | 2 +- .../helper_scripts/prepare_ccache_symlinks_rc | 41 +++++++++++++++++-- .../linux/grpc_run_interop_tests.sh | 3 ++ tools/run_tests/artifacts/package_targets.py | 4 +- tools/run_tests/run_tests.py | 4 +- 29 files changed, 340 insertions(+), 33 deletions(-) rename templates/tools/dockerfile/test/{csharp_buster_x64 => csharp_debian11_x64}/Dockerfile.template (94%) rename templates/tools/dockerfile/test/{ruby_buster_x64 => ruby_debian11_x64}/Dockerfile.template (92%) rename tools/dockerfile/test/{csharp_buster_x64 => csharp_debian11_x64}/Dockerfile (86%) rename tools/dockerfile/test/{ruby_buster_x64 => ruby_debian11_x64}/Dockerfile (84%) diff --git a/Rakefile b/Rakefile index 07e738f2113..189d6d5d5dd 100755 --- a/Rakefile +++ b/Rakefile @@ -113,12 +113,21 @@ task 'dlls', [:plat] do |t, args| out = GrpcBuildConfig::CORE_WINDOWS_DLL + # propagate env variables with ccache configuration to the rake-compiler-dock docker container + # and setup ccache symlinks as needed. + # TODO(jtattermusch): deduplicate creation of prepare_ccache_cmd + prepare_ccache_cmd = "export GRPC_BUILD_ENABLE_CCACHE=\"#{ENV.fetch('GRPC_BUILD_ENABLE_CCACHE', '')}\" && " + prepare_ccache_cmd += "export CCACHE_SECONDARY_STORAGE=\"#{ENV.fetch('CCACHE_SECONDARY_STORAGE', '')}\" && " + prepare_ccache_cmd += "export PATH=\"$PATH:/usr/local/bin\" && " + prepare_ccache_cmd += "source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc " + build_configs.each do |opt| env_comp = "CC=#{opt[:cross]}-gcc " env_comp += "CXX=#{opt[:cross]}-g++ " env_comp += "LD=#{opt[:cross]}-gcc " env_comp += "LDXX=#{opt[:cross]}-g++ " run_rake_compiler(opt[:platform], <<~EOT) + #{prepare_ccache_cmd} && \ gem update --system --no-document && \ #{env} #{env_comp} make -j#{nproc_override} #{out} && \ #{opt[:cross]}-strip -x -S #{out} && \ @@ -152,6 +161,13 @@ task 'gem:native', [:plat] do |t, args| # use env variable to set artifact build paralellism nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip + # propagate env variables with ccache configuration to the rake-compiler-dock docker container + # and setup ccache symlinks as needed. + prepare_ccache_cmd = "export GRPC_BUILD_ENABLE_CCACHE=\"#{ENV.fetch('GRPC_BUILD_ENABLE_CCACHE', '')}\" && " + prepare_ccache_cmd += "export CCACHE_SECONDARY_STORAGE=\"#{ENV.fetch('CCACHE_SECONDARY_STORAGE', '')}\" && " + prepare_ccache_cmd += "export PATH=\"$PATH:/usr/local/bin\" && " + prepare_ccache_cmd += "source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc " + supported_windows_platforms = ['x86-mingw32', 'x64-mingw32'] supported_unix_platforms = ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin'] supported_platforms = supported_windows_platforms + supported_unix_platforms @@ -178,6 +194,7 @@ task 'gem:native', [:plat] do |t, args| windows_platforms.each do |plat| run_rake_compiler(plat, <<~EOT) + #{prepare_ccache_cmd} && \ gem update --system --no-document && \ bundle && \ bundle exec rake clean && \ @@ -196,6 +213,7 @@ task 'gem:native', [:plat] do |t, args| unix_platforms.each do |plat| run_rake_compiler(plat, <<~EOT) + #{prepare_ccache_cmd} && \ gem update --system --no-document && \ bundle && \ bundle exec rake clean && \ diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template index 8ae3691e959..ab112a7adb1 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template @@ -14,13 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:buster + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> <%include file="../../csharp_deps.include"/> <%include file="../../csharp_dotnetcli_deps.include"/> + <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> + # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template index 8ae3691e959..ab112a7adb1 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template @@ -14,13 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:buster + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> <%include file="../../csharp_deps.include"/> <%include file="../../csharp_dotnetcli_deps.include"/> + <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> + # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile.template index 82dc1d16243..4dfe72e5e1c 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile.template @@ -14,13 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:jessie + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> <%include file="../../cxx_deps.include"/> - <%include file="../../cmake_jessie_backports.include"/> + <%include file="../../cmake.include"/> + <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> + # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template index 3bcc1b97ad6..0ab69691882 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template @@ -20,3 +20,6 @@ RUN ln -s $(which python3) /usr/bin/python <%include file="../../gcp_api_libraries.include"/> + + <%include file="../../cmake.include"/> + <%include file="../../ccache.include"/> \ No newline at end of file diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile.template index 6f528510ee3..e0c51ebe9b3 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile.template @@ -20,3 +20,6 @@ RUN ln -s $(which python3) /usr/bin/python <%include file="../../gcp_api_libraries.include"/> + + <%include file="../../cmake.include"/> + <%include file="../../ccache.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template index 67f8cd3b19f..e53ac1c7ff1 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:buster + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> <%include file="../../ruby_deps.include"/> + <%include file="../../cmake.include"/> + <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> + # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template b/templates/tools/dockerfile/test/csharp_debian11_x64/Dockerfile.template similarity index 94% rename from templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template rename to templates/tools/dockerfile/test/csharp_debian11_x64/Dockerfile.template index 0b647f4590e..016c074d5e6 100644 --- a/templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/csharp_debian11_x64/Dockerfile.template @@ -14,13 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:buster + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> <%include file="../../csharp_deps.include"/> <%include file="../../csharp_dotnetcli_deps.include"/> + <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> + # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/ruby_buster_x64/Dockerfile.template b/templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template similarity index 92% rename from templates/tools/dockerfile/test/ruby_buster_x64/Dockerfile.template rename to templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template index 2f05c70f479..a3fe030644a 100644 --- a/templates/tools/dockerfile/test/ruby_buster_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template @@ -14,15 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:buster + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> <%include file="../../ruby_deps.include"/> <%include file="../../cmake.include"/> + <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> - RUN python3 -m pip install virtualenv==16.7.9 - # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index e8afb5ee6e9..576f8473070 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:bullseye + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../cxx_deps.include"/> diff --git a/third_party/rake-compiler-dock/rake_x64-mingw32/Dockerfile b/third_party/rake-compiler-dock/rake_x64-mingw32/Dockerfile index 66637714d84..fd1f6dcf4ae 100644 --- a/third_party/rake-compiler-dock/rake_x64-mingw32/Dockerfile +++ b/third_party/rake-compiler-dock/rake_x64-mingw32/Dockerfile @@ -1,3 +1,17 @@ FROM larskanis/rake-compiler-dock-mri-x64-mingw32:1.1.0 RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz diff --git a/third_party/rake-compiler-dock/rake_x86-linux/Dockerfile b/third_party/rake-compiler-dock/rake_x86-linux/Dockerfile index 50aaae24150..812bd23e7b3 100644 --- a/third_party/rake-compiler-dock/rake_x86-linux/Dockerfile +++ b/third_party/rake-compiler-dock/rake_x86-linux/Dockerfile @@ -1 +1,15 @@ FROM larskanis/rake-compiler-dock-mri-x86-linux:1.1.0 + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz diff --git a/third_party/rake-compiler-dock/rake_x86-mingw32/Dockerfile b/third_party/rake-compiler-dock/rake_x86-mingw32/Dockerfile index 6a2880a9fee..ab855eb2251 100644 --- a/third_party/rake-compiler-dock/rake_x86-mingw32/Dockerfile +++ b/third_party/rake-compiler-dock/rake_x86-mingw32/Dockerfile @@ -1,3 +1,17 @@ FROM larskanis/rake-compiler-dock-mri-x86-mingw32:1.1.0 RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz diff --git a/third_party/rake-compiler-dock/rake_x86_64-linux/Dockerfile b/third_party/rake-compiler-dock/rake_x86_64-linux/Dockerfile index ede37a24083..78966e2dbcf 100644 --- a/third_party/rake-compiler-dock/rake_x86_64-linux/Dockerfile +++ b/third_party/rake-compiler-dock/rake_x86_64-linux/Dockerfile @@ -1 +1,15 @@ FROM larskanis/rake-compiler-dock-mri-x86_64-linux:1.1.0 + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz diff --git a/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile b/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile index aec40006ea6..b06af3b329d 100644 --- a/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile @@ -26,8 +26,36 @@ RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.6.13" "3.6.13" /opt RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.7.10" "3.7.10" /opt/python/cp37-cp37m RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.8.8" "3.8.8" /opt/python/cp38-cp38 RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.9.2" "3.9.2" /opt/python/cp39-cp39 -RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.9.2" "3.9.2" /opt/python/cp39-cp39 RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.10.0" "3.10.0rc1" /opt/python/cp310-cp310 ENV AUDITWHEEL_ARCH armv7l ENV AUDITWHEEL_PLAT linux_armv7l + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN unset CMAKE_TOOLCHAIN_FILE && unset AS AR CC CPP CXX LD \ + && curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + +# The dockcross base of this image sets CC and CXX to absolute paths, which makes it impossible to redirect their invocations +# to ccache via a symlink. Use relative paths instead. +ENV CC ${CROSS_TRIPLE}-gcc +ENV CXX ${CROSS_TRIPLE}-g++ + +# For historical reasons, task_runner.py the script under docker container using "bash -l" +# which loads /etc/profile on startup. dockcross/linux-armv7 is based on an image where +# /etc/profile overwrites contents of PATH (for security reasons) when run as root. +# That causes the crosscompiler binaries located under /usr/xcc/armv7-unknown-linux-gnueabi/bin +# to be removed from PATH. Since in our builds we don't need the /etc/profile for anything, we can just +# truncate it. +# TODO(jtattermusch): Remove this hack when possible. +RUN echo "# file contents removed to avoid resetting PATH set by the docker image" >/etc/profile diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile index 1aee9c12f84..d249c03c38d 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile @@ -25,9 +25,29 @@ RUN /opt/python/cp38-cp38/bin/pip install --upgrade cython RUN /opt/python/cp39-cp39/bin/pip install --upgrade cython RUN /opt/python/cp310-cp310/bin/pip install --upgrade cython +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN unset CMAKE_TOOLCHAIN_FILE && unset AS AR CC CPP CXX LD \ + && curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + # the dockcross docker image sets variables like CC, CXX etc. # to point to the crosscompilation toolchain, but doesn't set corresponding # variables for the "strip" and "objcopy" tools. # see https://github.com/dockcross/dockcross/blob/4349cb4999401cbf22a90f46f5052d29be240e50/manylinux2014-aarch64/Dockerfile.in#L23 ENV STRIP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip \ OBJCOPY=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-objcopy + +# The dockcross base of this image sets CC and CXX to absolute paths, which makes it impossible to redirect their invocations +# to ccache via a symlink. Use relative paths instead. +ENV CC ${CROSS_TRIPLE}-gcc +ENV CXX ${CROSS_TRIPLE}-g++ diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 6b58b964f82..a22c69a4049 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:buster +FROM debian:11 #================= # Basic C core dependencies @@ -119,8 +119,23 @@ RUN mkdir warmup \ && cd .. \ && rm -rf warmup +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index 6b58b964f82..a22c69a4049 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:buster +FROM debian:11 #================= # Basic C core dependencies @@ -119,8 +119,23 @@ RUN mkdir warmup \ && cd .. \ && rm -rf warmup +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index e59499f80e6..7722bd891e5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:jessie +FROM debian:11 #================= # Basic C core dependencies @@ -85,16 +85,28 @@ RUN python3 -m pip install --upgrade google-auth==1.23.0 google-api-python-clien RUN apt-get update && apt-get -y install libc++-dev clang && apt-get clean #================= -# Use cmake 3.6 from jessie-backports -# should only be used for images based on debian jessie. +# Install cmake +# Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. -RUN echo "deb http://archive.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list -RUN echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf -RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list -RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean +RUN apt-get update && apt-get install -y cmake && apt-get clean + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 7445b7112df..b42fc13848c 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -67,3 +67,23 @@ RUN ln -s $(which python3) /usr/bin/python # These are needed for uploading test results to BigQuery (e.g. by tools/run_tests scripts) RUN python3 -m pip install --upgrade google-auth==1.23.0 google-api-python-client==1.12.8 oauth2client==4.1.0 + +#================= +# Install cmake +# Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. + +RUN apt-get update && apt-get install -y cmake && apt-get clean + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz diff --git a/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile index 14961a678c8..2b481570932 100644 --- a/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile @@ -67,3 +67,24 @@ RUN ln -s $(which python3) /usr/bin/python # These are needed for uploading test results to BigQuery (e.g. by tools/run_tests scripts) RUN python3 -m pip install --upgrade google-auth==1.23.0 google-api-python-client==1.12.8 oauth2client==4.1.0 + +#================= +# Install cmake +# Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. + +RUN apt-get update && apt-get install -y cmake && apt-get clean + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index b121504da5f..d05ee85bf14 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:buster +FROM debian:11 #================= # Basic C core dependencies @@ -97,8 +97,29 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.5' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document -v 1.9" +#================= +# Install cmake +# Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. + +RUN apt-get update && apt-get install -y cmake && apt-get clean + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/test/csharp_buster_x64/Dockerfile b/tools/dockerfile/test/csharp_debian11_x64/Dockerfile similarity index 86% rename from tools/dockerfile/test/csharp_buster_x64/Dockerfile rename to tools/dockerfile/test/csharp_debian11_x64/Dockerfile index 774ce0840c9..952d65db8c1 100644 --- a/tools/dockerfile/test/csharp_buster_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_debian11_x64/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:buster +FROM debian:11 #================= # Basic C core dependencies @@ -119,8 +119,23 @@ RUN mkdir warmup \ && cd .. \ && rm -rf warmup +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/test/ruby_buster_x64/Dockerfile b/tools/dockerfile/test/ruby_debian11_x64/Dockerfile similarity index 84% rename from tools/dockerfile/test/ruby_buster_x64/Dockerfile rename to tools/dockerfile/test/ruby_debian11_x64/Dockerfile index 912a89137af..d05ee85bf14 100644 --- a/tools/dockerfile/test/ruby_buster_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_debian11_x64/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:buster +FROM debian:11 #================= # Basic C core dependencies @@ -103,11 +103,23 @@ RUN /bin/bash -l -c "gem install bundler --no-document -v 1.9" RUN apt-get update && apt-get install -y cmake && apt-get clean +#================= +# Install ccache -RUN mkdir /var/local/jenkins +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz -RUN python3 -m pip install virtualenv==16.7.9 +RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 2ade5228494..665602d4c21 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:bullseye +FROM debian:11 #================= # Basic C core dependencies diff --git a/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc b/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc index eb666d53107..36e5a898f59 100644 --- a/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc +++ b/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc @@ -20,12 +20,47 @@ if [ "${GRPC_BUILD_ENABLE_CCACHE}" != "" ] then if [ -x "$(command -v ccache)" ] then + SUPPORTED_COMPILERS=( + # common compiler binaries + "gcc" + "g++" + "clang" + "clang++" + "cc" + "c++" + # ruby artifacts: rake-compiler-dock crosscompilers + # TODO(jtattermusch): ensure that the list of ruby crosscompilers stays up to date. + "x86_64-redhat-linux-gcc" + "x86_64-redhat-linux-g++" + "i686-redhat-linux-gcc" + "i686-redhat-linux-g++" + "x86_64-w64-mingw32-gcc" + "x86_64-w64-mingw32-g++" + "i686-w64-mingw32-gcc" + "i686-w64-mingw32-g++" + "x86_64-apple-darwin-clang" + "x86_64-apple-darwin-clang++" + "aarch64-apple-darwin-clang" + "aarch64-apple-darwin-clang++" + # python artifacts: dockcross crosscompilers + "aarch64-unknown-linux-gnueabi-gcc" + "aarch64-unknown-linux-gnueabi-g++" + "armv7-unknown-linux-gnueabi-gcc" + "armv7-unknown-linux-gnueabi-g++" + ) + CCACHE_BINARY_PATH="$(command -v ccache)" TEMP_CCACHE_BINDIR="$(mktemp -d)" - # TODO: symlinks for crosscompilers... - for compiler in "gcc" "g++" "clang" "clang++" "cc" "c++" + + for compiler in "${SUPPORTED_COMPILERS[@]}" do - ln -s "$(which ccache)" "${TEMP_CCACHE_BINDIR}/${compiler}" + # create a symlink pointing to ccache if compiler binary exists + if [ -x "$(command -v $compiler)" ] + then + ln -s "${CCACHE_BINARY_PATH}" "${TEMP_CCACHE_BINDIR}/${compiler}" + echo "Creating symlink $compiler pointing to ${CCACHE_BINARY_PATH}" + fi done + echo "Adding ${TEMP_CCACHE_BINDIR} to PATH" export PATH="${TEMP_CCACHE_BINDIR}:$PATH" fi fi diff --git a/tools/internal_ci/linux/grpc_run_interop_tests.sh b/tools/internal_ci/linux/grpc_run_interop_tests.sh index 8b02bd05cff..0ebcf3849cb 100755 --- a/tools/internal_ci/linux/grpc_run_interop_tests.sh +++ b/tools/internal_ci/linux/grpc_run_interop_tests.sh @@ -26,4 +26,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc source tools/internal_ci/helper_scripts/prepare_build_interop_rc +# configure ccache +source tools/internal_ci/helper_scripts/prepare_ccache_rc + tools/run_tests/run_interop_tests.py $RUN_TESTS_FLAGS diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index daa19cc5a86..23be1701cce 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -96,13 +96,13 @@ class CSharpPackage: if self.unity: return create_docker_jobspec( self.name, - 'tools/dockerfile/test/csharp_buster_x64', + 'tools/dockerfile/test/csharp_debian11_x64', 'src/csharp/build_unitypackage.sh', environ=environ) else: return create_docker_jobspec( self.name, - 'tools/dockerfile/test/csharp_buster_x64', + 'tools/dockerfile/test/csharp_debian11_x64', 'src/csharp/build_nuget.sh', environ=environ) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0c9bbd2e7b9..e74fdab463d 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -861,7 +861,7 @@ class RubyLanguage(object): return 'Makefile' def dockerfile_dir(self): - return 'tools/dockerfile/test/ruby_buster_%s' % _docker_arch_suffix( + return 'tools/dockerfile/test/ruby_debian11_%s' % _docker_arch_suffix( self.args.arch) def __str__(self): @@ -888,7 +888,7 @@ class CSharpLanguage(object): _check_arch(self.args.arch, ['default']) self._cmake_arch_option = 'x64' else: - self._docker_distro = 'buster' + self._docker_distro = 'debian11' def test_specs(self): with open('src/csharp/tests.json') as f: