Introduce C# single-platform nuget build (and distribtests) (#27180)

* C# linux distribtests

* add distribtests C# linux job cfg

* support running a subset of C# artifact/distribtests on jobs

src/csharp/build_nuget.sh

@@ -28,7 +28,31 @@ mkdir -p protoc_plugins
 cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/protoc_* protoc_plugins || true
 
 # Add current timestamp to dev nugets
-./expand_dev_version.sh
+./nuget_helpers/expand_dev_version.sh
+
+# For building the nugets we normally need native libraries and binaries
+# built on multiple different platforms (linux, mac, windows), which makes
+# it difficult to support a local build of the nuget.
+# To allow simple local builds (restricted to a single platform),
+# we provide a way of building "partial" nugets that only include artifacts
+# that can be built locally on a given platform (e.g. linux), and
+# contain placeholders (empty files) for artifacts that normally need
+# to be built on a different platform. Because such nugets obviously
+# only work on a single platform (and are broken on other platform),
+# whenever we are building such nugets, we clearly mark them as
+# "singleplatform only" to avoid mixing them up with the full "multiplatform"
+# nugets by accident.
+if [ "${GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET}" != "" ]
+then
+  # create placeholders for artifacts that can't be built
+  # on the current platform.
+  ./nuget_helpers/create_fake_native_artifacts.sh || true
+
+  # add a suffix to the nuget's version
+  # to avoid confusing the package with a full nuget package.
+  # NOTE: adding the suffix must be done AFTER expand_dev_version.sh has run.
+  sed -ibak "s/<\/GrpcCsharpVersion>/.singleplatform<\/GrpcCsharpVersion>/" build/dependencies.props
+fi
 
 dotnet restore Grpc.sln
 
@@ -44,7 +68,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../artifacts
 dotnet pack --configuration Release Grpc.HealthCheck --output ../../artifacts
 dotnet pack --configuration Release Grpc.Reflection --output ../../artifacts
 dotnet pack --configuration Release Grpc.Tools --output ../../artifacts
-# rem build auxiliary packages
+# build auxiliary packages
 dotnet pack --configuration Release Grpc --output ../../artifacts
 dotnet pack --configuration Release Grpc.Core.NativeDebug --output ../../artifacts
 dotnet pack --configuration Release Grpc.Core.Xamarin --output ../../artifacts

src/csharp/build_unitypackage.sh

@@ -24,7 +24,7 @@ mkdir -p nativelibs
 cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/csharp_ext_* nativelibs || true
 
 # Add current timestamp to dev nugets
-./expand_dev_version.sh
+./nuget_helpers/expand_dev_version.sh
 
 # Extract current Grpc.Core version from build/dependencies.props
 UNITYPACKAGE_VERSION="$(grep -o '<GrpcCsharpVersion>.*</GrpcCsharpVersion>' build/dependencies.props | sed 's/<GrpcCsharpVersion>//' | sed 's/<\/GrpcCsharpVersion>//')"

src/csharp/nuget_helpers/create_fake_native_artifacts.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Create placeholders ("fake artifacts") for native binaries that
+# can't be built locally on current platform.
+
+set -ex
+
+cd "$(dirname "$0")/../../.."
+
+cd src/csharp
+
+# Create fake grpc_csharp_ext artifacts
+mkdir -p nativelibs
+pushd nativelibs
+
+if [[ "$(uname)" != "Linux" ]]
+then
+   mkdir -p csharp_ext_linux_x64
+   touch csharp_ext_linux_x64/libgrpc_csharp_ext.so
+   touch csharp_ext_linux_x64/libgrpc_csharp_ext.dbginfo.so
+
+   mkdir -p csharp_ext_linux_x86
+   touch csharp_ext_linux_x86/libgrpc_csharp_ext.so
+
+   mkdir -p csharp_ext_linux_aarch64
+   touch csharp_ext_linux_aarch64/libgrpc_csharp_ext.so
+   touch csharp_ext_linux_aarch64/libgrpc_csharp_ext.dbginfo.so
+
+   mkdir -p csharp_ext_linux_android_armeabi-v7a
+   touch csharp_ext_linux_android_armeabi-v7a/libgrpc_csharp_ext.so
+
+   mkdir -p csharp_ext_linux_android_arm64-v8a
+   touch csharp_ext_linux_android_arm64-v8a/libgrpc_csharp_ext.so
+
+   mkdir -p csharp_ext_linux_android_x86
+   touch csharp_ext_linux_android_x86/libgrpc_csharp_ext.so
+fi
+
+if [[ "$(uname)" != "Darwin" ]]
+then
+    mkdir -p csharp_ext_macos_x64
+    touch csharp_ext_macos_x64/libgrpc_csharp_ext.dylib
+
+    mkdir -p csharp_ext_macos_ios
+    touch csharp_ext_macos_ios/libgrpc_csharp_ext.a
+    touch csharp_ext_macos_ios/libgrpc.a
+fi
+
+if [[ "$(uname)" != "WindowsNT" ]]
+then
+    mkdir -p csharp_ext_windows_x64
+    touch csharp_ext_windows_x64/grpc_csharp_ext.dll
+    touch csharp_ext_windows_x64/grpc_csharp_ext.pdb
+
+    mkdir -p csharp_ext_windows_x86
+    touch csharp_ext_windows_x86/grpc_csharp_ext.dll
+    touch csharp_ext_windows_x86/grpc_csharp_ext.pdb
+fi
+
+popd
+
+# Create fake protoc artifacts
+mkdir -p protoc_plugins
+pushd protoc_plugins
+
+if [[ "$(uname)" != "Linux" ]]
+then
+  mkdir -p protoc_linux_x64
+  touch protoc_linux_x64/protoc
+  touch protoc_linux_x64/grpc_csharp_plugin
+
+  mkdir -p protoc_linux_x86
+  touch protoc_linux_x86/protoc
+  touch protoc_linux_x86/grpc_csharp_plugin
+
+  mkdir -p protoc_linux_aarch64
+  touch protoc_linux_aarch64/protoc
+  touch protoc_linux_aarch64/grpc_csharp_plugin
+fi
+
+if [[ "$(uname)" != "Darwin" ]]
+then
+    mkdir -p protoc_macos_x64
+    touch protoc_macos_x64/protoc
+    touch protoc_macos_x64/grpc_csharp_plugin
+fi
+
+if [[ "$(uname)" != "WindowsNT" ]]
+then
+    mkdir -p protoc_windows_x86
+    touch protoc_windows_x86/protoc.exe
+    touch protoc_windows_x86/grpc_csharp_plugin.exe
+
+    mkdir -p protoc_windows_x64
+    touch protoc_windows_x64/protoc.exe
+    touch protoc_windows_x64/grpc_csharp_plugin.exe
+fi
+
+popd

src/csharp/nuget_helpers/expand_dev_version.sh

@@ -18,7 +18,7 @@
 
 set -e
 
-cd "$(dirname "$0")"
+cd "$(dirname "$0")/.."
 
 DEV_DATETIME_SUFFIX=$(date -u "+%Y%m%d%H%M")
 # expand the -dev suffix to contain current timestamp

tools/internal_ci/linux/grpc_distribtests_csharp.cfg

@@ -0,0 +1,26 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_csharp.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/linux/grpc_distribtests_csharp.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+# some distribtests use a pre-registered binfmt_misc hook
+# to automatically execute foreign binaries (such as aarch64)
+# under qemu emulator.
+source tools/internal_ci/helper_scripts/prepare_qemu_rc
+
+# Build all C# linux artifacts
+tools/run_tests/task_runner.py -f artifact linux csharp ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts_csharp/sponge_log.xml || FAILED="true"
+
+# Build all protoc linux artifacts
+tools/run_tests/task_runner.py -f artifact linux protoc ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts_protoc/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+
+# This step builds the nuget packages from input_artifacts
+# Set env variable option to build single platform version of the nugets.
+# (this is required as we only have the linux artifacts at hand)
+GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET=1 tools/run_tests/task_runner.py -f package linux csharp nuget -j 2 -x build_packages/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+# in addition to that, preserve the contents of "artifacts" directory since we want kokoro
+# to upload its contents as job output artifacts
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+
+# Run all C# linux distribtests
+# We run the distribtests even if some of the artifacts have failed to build, since that gives
+# a better signal about which distribtest are affected by the currently broken artifact builds.
+tools/run_tests/task_runner.py -f distribtest linux csharp ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+
+tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/internal_ci/linux/pull_request/grpc_distribtests_csharp.cfg

@@ -0,0 +1,31 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_csharp.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "presubmit"
+}

tools/run_tests/artifacts/package_targets.py

@@ -88,14 +88,22 @@ class CSharpPackage:
         return []
 
     def build_jobspec(self):
+        environ = {
+            'GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET':
+                os.getenv('GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET')
+        }
         if self.unity:
             return create_docker_jobspec(
-                self.name, 'tools/dockerfile/test/csharp_buster_x64',
-                'src/csharp/build_unitypackage.sh')
+                self.name,
+                'tools/dockerfile/test/csharp_buster_x64',
+                'src/csharp/build_unitypackage.sh',
+                environ=environ)
         else:
             return create_docker_jobspec(
-                self.name, 'tools/dockerfile/test/csharp_buster_x64',
-                'src/csharp/build_nuget.sh')
+                self.name,
+                'tools/dockerfile/test/csharp_buster_x64',
+                'src/csharp/build_nuget.sh',
+                environ=environ)
 
     def __str__(self):
         return self.name