diff --git a/src/csharp/build_nuget.sh b/src/csharp/build_nuget.sh index 42af0c897bd..367320dccc0 100755 --- a/src/csharp/build_nuget.sh +++ b/src/csharp/build_nuget.sh @@ -28,7 +28,31 @@ mkdir -p protoc_plugins cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/protoc_* protoc_plugins || true # Add current timestamp to dev nugets -./expand_dev_version.sh +./nuget_helpers/expand_dev_version.sh + +# For building the nugets we normally need native libraries and binaries +# built on multiple different platforms (linux, mac, windows), which makes +# it difficult to support a local build of the nuget. +# To allow simple local builds (restricted to a single platform), +# we provide a way of building "partial" nugets that only include artifacts +# that can be built locally on a given platform (e.g. linux), and +# contain placeholders (empty files) for artifacts that normally need +# to be built on a different platform. Because such nugets obviously +# only work on a single platform (and are broken on other platform), +# whenever we are building such nugets, we clearly mark them as +# "singleplatform only" to avoid mixing them up with the full "multiplatform" +# nugets by accident. +if [ "${GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET}" != "" ] +then + # create placeholders for artifacts that can't be built + # on the current platform. + ./nuget_helpers/create_fake_native_artifacts.sh || true + + # add a suffix to the nuget's version + # to avoid confusing the package with a full nuget package. + # NOTE: adding the suffix must be done AFTER expand_dev_version.sh has run. + sed -ibak "s/<\/GrpcCsharpVersion>/.singleplatform<\/GrpcCsharpVersion>/" build/dependencies.props +fi dotnet restore Grpc.sln @@ -44,7 +68,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../artifacts dotnet pack --configuration Release Grpc.Tools --output ../../artifacts -# rem build auxiliary packages +# build auxiliary packages dotnet pack --configuration Release Grpc --output ../../artifacts dotnet pack --configuration Release Grpc.Core.NativeDebug --output ../../artifacts dotnet pack --configuration Release Grpc.Core.Xamarin --output ../../artifacts diff --git a/src/csharp/build_unitypackage.sh b/src/csharp/build_unitypackage.sh index 4363ee1955a..e4aa89612a7 100755 --- a/src/csharp/build_unitypackage.sh +++ b/src/csharp/build_unitypackage.sh @@ -24,7 +24,7 @@ mkdir -p nativelibs cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/csharp_ext_* nativelibs || true # Add current timestamp to dev nugets -./expand_dev_version.sh +./nuget_helpers/expand_dev_version.sh # Extract current Grpc.Core version from build/dependencies.props UNITYPACKAGE_VERSION="$(grep -o '.*' build/dependencies.props | sed 's///' | sed 's/<\/GrpcCsharpVersion>//')" diff --git a/src/csharp/nuget_helpers/create_fake_native_artifacts.sh b/src/csharp/nuget_helpers/create_fake_native_artifacts.sh new file mode 100755 index 00000000000..cc68b2e652f --- /dev/null +++ b/src/csharp/nuget_helpers/create_fake_native_artifacts.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create placeholders ("fake artifacts") for native binaries that +# can't be built locally on current platform. + +set -ex + +cd "$(dirname "$0")/../../.." + +cd src/csharp + +# Create fake grpc_csharp_ext artifacts +mkdir -p nativelibs +pushd nativelibs + +if [[ "$(uname)" != "Linux" ]] +then + mkdir -p csharp_ext_linux_x64 + touch csharp_ext_linux_x64/libgrpc_csharp_ext.so + touch csharp_ext_linux_x64/libgrpc_csharp_ext.dbginfo.so + + mkdir -p csharp_ext_linux_x86 + touch csharp_ext_linux_x86/libgrpc_csharp_ext.so + + mkdir -p csharp_ext_linux_aarch64 + touch csharp_ext_linux_aarch64/libgrpc_csharp_ext.so + touch csharp_ext_linux_aarch64/libgrpc_csharp_ext.dbginfo.so + + mkdir -p csharp_ext_linux_android_armeabi-v7a + touch csharp_ext_linux_android_armeabi-v7a/libgrpc_csharp_ext.so + + mkdir -p csharp_ext_linux_android_arm64-v8a + touch csharp_ext_linux_android_arm64-v8a/libgrpc_csharp_ext.so + + mkdir -p csharp_ext_linux_android_x86 + touch csharp_ext_linux_android_x86/libgrpc_csharp_ext.so +fi + +if [[ "$(uname)" != "Darwin" ]] +then + mkdir -p csharp_ext_macos_x64 + touch csharp_ext_macos_x64/libgrpc_csharp_ext.dylib + + mkdir -p csharp_ext_macos_ios + touch csharp_ext_macos_ios/libgrpc_csharp_ext.a + touch csharp_ext_macos_ios/libgrpc.a +fi + +if [[ "$(uname)" != "WindowsNT" ]] +then + mkdir -p csharp_ext_windows_x64 + touch csharp_ext_windows_x64/grpc_csharp_ext.dll + touch csharp_ext_windows_x64/grpc_csharp_ext.pdb + + mkdir -p csharp_ext_windows_x86 + touch csharp_ext_windows_x86/grpc_csharp_ext.dll + touch csharp_ext_windows_x86/grpc_csharp_ext.pdb +fi + +popd + +# Create fake protoc artifacts +mkdir -p protoc_plugins +pushd protoc_plugins + +if [[ "$(uname)" != "Linux" ]] +then + mkdir -p protoc_linux_x64 + touch protoc_linux_x64/protoc + touch protoc_linux_x64/grpc_csharp_plugin + + mkdir -p protoc_linux_x86 + touch protoc_linux_x86/protoc + touch protoc_linux_x86/grpc_csharp_plugin + + mkdir -p protoc_linux_aarch64 + touch protoc_linux_aarch64/protoc + touch protoc_linux_aarch64/grpc_csharp_plugin +fi + +if [[ "$(uname)" != "Darwin" ]] +then + mkdir -p protoc_macos_x64 + touch protoc_macos_x64/protoc + touch protoc_macos_x64/grpc_csharp_plugin +fi + +if [[ "$(uname)" != "WindowsNT" ]] +then + mkdir -p protoc_windows_x86 + touch protoc_windows_x86/protoc.exe + touch protoc_windows_x86/grpc_csharp_plugin.exe + + mkdir -p protoc_windows_x64 + touch protoc_windows_x64/protoc.exe + touch protoc_windows_x64/grpc_csharp_plugin.exe +fi + +popd diff --git a/src/csharp/expand_dev_version.sh b/src/csharp/nuget_helpers/expand_dev_version.sh similarity index 97% rename from src/csharp/expand_dev_version.sh rename to src/csharp/nuget_helpers/expand_dev_version.sh index 714762afac1..6edae5940b8 100755 --- a/src/csharp/expand_dev_version.sh +++ b/src/csharp/nuget_helpers/expand_dev_version.sh @@ -18,7 +18,7 @@ set -e -cd "$(dirname "$0")" +cd "$(dirname "$0")/.." DEV_DATETIME_SUFFIX=$(date -u "+%Y%m%d%H%M") # expand the -dev suffix to contain current timestamp diff --git a/tools/internal_ci/linux/grpc_distribtests_csharp.cfg b/tools/internal_ci/linux/grpc_distribtests_csharp.cfg new file mode 100644 index 00000000000..cff345afa27 --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests_csharp.cfg @@ -0,0 +1,26 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_csharp.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_distribtests_csharp.sh b/tools/internal_ci/linux/grpc_distribtests_csharp.sh new file mode 100755 index 00000000000..602807f1237 --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests_csharp.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +# some distribtests use a pre-registered binfmt_misc hook +# to automatically execute foreign binaries (such as aarch64) +# under qemu emulator. +source tools/internal_ci/helper_scripts/prepare_qemu_rc + +# Build all C# linux artifacts +tools/run_tests/task_runner.py -f artifact linux csharp ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts_csharp/sponge_log.xml || FAILED="true" + +# Build all protoc linux artifacts +tools/run_tests/task_runner.py -f artifact linux protoc ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts_protoc/sponge_log.xml || FAILED="true" + +# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/* input_artifacts/ || true + +# This step builds the nuget packages from input_artifacts +# Set env variable option to build single platform version of the nugets. +# (this is required as we only have the linux artifacts at hand) +GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET=1 tools/run_tests/task_runner.py -f package linux csharp nuget -j 2 -x build_packages/sponge_log.xml || FAILED="true" + +# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +# in addition to that, preserve the contents of "artifacts" directory since we want kokoro +# to upload its contents as job output artifacts +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/* input_artifacts/ || true + +# Run all C# linux distribtests +# We run the distribtests even if some of the artifacts have failed to build, since that gives +# a better signal about which distribtest are affected by the currently broken artifact builds. +tools/run_tests/task_runner.py -f distribtest linux csharp ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" + +tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_csharp.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_csharp.cfg new file mode 100644 index 00000000000..fa9cf177605 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_csharp.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_csharp.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 1103c70a8cc..25dcde25ae1 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -88,14 +88,22 @@ class CSharpPackage: return [] def build_jobspec(self): + environ = { + 'GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET': + os.getenv('GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET') + } if self.unity: return create_docker_jobspec( - self.name, 'tools/dockerfile/test/csharp_buster_x64', - 'src/csharp/build_unitypackage.sh') + self.name, + 'tools/dockerfile/test/csharp_buster_x64', + 'src/csharp/build_unitypackage.sh', + environ=environ) else: return create_docker_jobspec( - self.name, 'tools/dockerfile/test/csharp_buster_x64', - 'src/csharp/build_nuget.sh') + self.name, + 'tools/dockerfile/test/csharp_buster_x64', + 'src/csharp/build_nuget.sh', + environ=environ) def __str__(self): return self.name