Upgrade dotnet SDK 3 for C# build (#22964)

* upgrade dotnet SDK 3 for C# build

* add C# global.json

* install dotnet SDK 3 on macos

* upgrade C# docker images to debian buster

* regenerate dockerfiles

src/csharp/global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "3.0.100",
+    "rollForward": "latestMinor"
+  }
+}

src/csharp/install_dotnet_sdk.ps1

@@ -19,4 +19,8 @@ $InstallScriptPath = Join-Path  "$env:TEMP" 'dotnet-install.ps1'
 # Download install script
 Write-Host "Downloading install script: $InstallScriptUrl => $InstallScriptPath"
 Invoke-WebRequest -Uri $InstallScriptUrl -OutFile $InstallScriptPath
-&$InstallScriptPath -Version 2.1.504
+
+# Installed versions should be kept in sync with
+# templates/tools/dockerfile/csharp_dotnetcli_deps.include
+&$InstallScriptPath -Version 2.1.802
+&$InstallScriptPath -Version 3.1.301

templates/tools/dockerfile/csharp_deps.include

@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y ${'\\'}
     mono-devel ${'\\'}
     ca-certificates-mono ${'\\'}

templates/tools/dockerfile/csharp_dotnetcli_deps.include

@@ -1,20 +1,17 @@
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 3.1.301
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz ${'\\'}
     && mkdir -p /usr/share/dotnet ${'\\'}
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet ${'\\'}
     && rm dotnet.tar.gz ${'\\'}
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz ${'\\'}
-    && mkdir -p dotnet_old ${'\\'}
-    && tar zxf dotnet_old.tar.gz -C dotnet_old ${'\\'}
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ ${'\\'}
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
+# dotnet SDK 2.1 required to run netcoreapp2.1 targets
+ENV DOTNET_SDK_OLD_VERSION 2.1.802
+RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_OLD_VERSION/dotnet-sdk-$DOTNET_SDK_OLD_VERSION-linux-x64.tar.gz ${'\\'}
+    && mkdir -p /usr/share/dotnet ${'\\'}
+    && tar -zxf dotnet_old.tar.gz -C /usr/share/dotnet ${'\\'}
+    && rm dotnet_old.tar.gz
 
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip

templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:stretch
+  FROM debian:buster
   
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../python_deps.include"/>

templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:stretch
+  FROM debian:buster
   
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../python_deps.include"/>

templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:stretch
+  FROM debian:buster
   
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../python_deps.include"/>

tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:buster
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -74,7 +74,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y \
     mono-devel \
     ca-certificates-mono \
@@ -82,22 +82,19 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean
 
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 3.1.301
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \
     && mkdir -p /usr/share/dotnet \
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \
     && rm dotnet.tar.gz \
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \
-    && mkdir -p dotnet_old \
-    && tar zxf dotnet_old.tar.gz -C dotnet_old \
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
+# dotnet SDK 2.1 required to run netcoreapp2.1 targets
+ENV DOTNET_SDK_OLD_VERSION 2.1.802
+RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_OLD_VERSION/dotnet-sdk-$DOTNET_SDK_OLD_VERSION-linux-x64.tar.gz \
+    && mkdir -p /usr/share/dotnet \
+    && tar -zxf dotnet_old.tar.gz -C /usr/share/dotnet \
+    && rm dotnet_old.tar.gz
 
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip

tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:buster
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -74,7 +74,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y \
     mono-devel \
     ca-certificates-mono \
@@ -82,22 +82,19 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean
 
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 3.1.301
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \
     && mkdir -p /usr/share/dotnet \
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \
     && rm dotnet.tar.gz \
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \
-    && mkdir -p dotnet_old \
-    && tar zxf dotnet_old.tar.gz -C dotnet_old \
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
+# dotnet SDK 2.1 required to run netcoreapp2.1 targets
+ENV DOTNET_SDK_OLD_VERSION 2.1.802
+RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_OLD_VERSION/dotnet-sdk-$DOTNET_SDK_OLD_VERSION-linux-x64.tar.gz \
+    && mkdir -p /usr/share/dotnet \
+    && tar -zxf dotnet_old.tar.gz -C /usr/share/dotnet \
+    && rm dotnet_old.tar.gz
 
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip

tools/dockerfile/test/csharp_buster_x64/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:buster
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y \
     mono-devel \
     ca-certificates-mono \
@@ -85,22 +85,19 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean
 
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 3.1.301
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \
     && mkdir -p /usr/share/dotnet \
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \
     && rm dotnet.tar.gz \
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \
-    && mkdir -p dotnet_old \
-    && tar zxf dotnet_old.tar.gz -C dotnet_old \
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
+# dotnet SDK 2.1 required to run netcoreapp2.1 targets
+ENV DOTNET_SDK_OLD_VERSION 2.1.802
+RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_OLD_VERSION/dotnet-sdk-$DOTNET_SDK_OLD_VERSION-linux-x64.tar.gz \
+    && mkdir -p /usr/share/dotnet \
+    && tar -zxf dotnet_old.tar.gz -C /usr/share/dotnet \
+    && rm dotnet_old.tar.gz
 
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip

tools/internal_ci/helper_scripts/prepare_build_macos_rc

@@ -127,6 +127,11 @@ fi
 
 if [ "${PREPARE_BUILD_INSTALL_DEPS_CSHARP}" == "true" ]
 then
+  # install dotnet SDK 3.x
+  # no need to install dotnet SDK 2.x since it's already installed on Kokoro MacOS workers.
+  time curl -sSL -O https://download.visualstudio.microsoft.com/download/pr/964ae449-a8b8-46d1-b944-c54f6e1bf8fc/f0cbcb2df3409d865b62f0c02a9ebbb9/dotnet-sdk-3.1.409-osx-x64.pkg
+  time sudo installer -pkg ./dotnet-sdk-3.1.409-osx-x64.pkg -target /
+
   # Disable some unwanted dotnet options
   export NUGET_XMLDOC_MODE=skip
   export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true

tools/run_tests/run_tests.py

@@ -922,7 +922,7 @@ class CSharpLanguage(object):
             self._cmake_arch_option = 'x64'
         else:
             _check_compiler(self.args.compiler, ['default', 'coreclr'])
-            self._docker_distro = 'stretch'
+            self._docker_distro = 'buster'
 
     def test_specs(self):
         with open('src/csharp/tests.json') as f: