Add back C# distribtests for Grpc.Tools (#31399)

* remove remains of grpc_csharp_ext artifact build completely

* add back part of C# distribtests

* redirect C# linux distribtests

* comment out some distribtests

* switch updateversion to Grpc.tools

* fix .proto

* fix distribtest complile

* temporarily comment out service in .proto

* cleanup

* improvements

* reenable mac distribtests

* fixes
pull/31411/head
Jan Tattermusch 2 years ago committed by GitHub
parent ed1bad547c
commit 50bad293c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      test/distrib/csharp/.gitignore
  2. 3
      test/distrib/csharp/DistribTest/.gitignore
  3. 29
      test/distrib/csharp/DistribTest/DistribTestDotNet.csproj
  4. 63
      test/distrib/csharp/DistribTest/Program.cs
  5. 23
      test/distrib/csharp/DistribTest/duplicate_proto/testcodegen.proto
  6. 29
      test/distrib/csharp/DistribTest/testcodegen.proto
  7. 8
      test/distrib/csharp/NuGet.Config
  8. 81
      test/distrib/csharp/run_distrib_test_dotnetcli.sh
  9. 31
      test/distrib/csharp/update_version.sh
  10. 3
      tools/internal_ci/linux/grpc_distribtests_csharp.sh
  11. 3
      tools/internal_ci/macos/grpc_distribtests_csharp.sh
  12. 3
      tools/internal_ci/windows/grpc_distribtests_csharp.bat
  13. 86
      tools/run_tests/artifacts/artifact_targets.py
  14. 16
      tools/run_tests/artifacts/build_artifact_csharp.bat
  15. 19
      tools/run_tests/artifacts/build_artifact_csharp.sh
  16. 19
      tools/run_tests/artifacts/build_artifact_csharp_android.sh
  17. 19
      tools/run_tests/artifacts/build_artifact_csharp_ios.sh
  18. 36
      tools/run_tests/artifacts/distribtest_targets.py

@ -0,0 +1,6 @@
packages
*.userprefs
*.csproj.user
*.suo
/TestNugetFeed

@ -0,0 +1,3 @@
bin
obj
*.lock.json

@ -0,0 +1,29 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<!-- set TargetFrameworks, but allow skipping selected frameworks by setting env variables -->
<TargetFrameworks></TargetFrameworks>
<TargetFrameworks Condition="'$(SKIP_NET45_DISTRIBTEST)' == ''">$(TargetFrameworks);net45</TargetFrameworks>
<TargetFrameworks Condition="'$(SKIP_NETCOREAPP21_DISTRIBTEST)' == ''">$(TargetFrameworks);netcoreapp2.1</TargetFrameworks>
<TargetFrameworks Condition="'$(SKIP_NETCOREAPP31_DISTRIBTEST)' == ''">$(TargetFrameworks);netcoreapp3.1</TargetFrameworks>
<TargetFrameworks Condition="'$(SKIP_NET50_DISTRIBTEST)' == ''">$(TargetFrameworks);net5.0</TargetFrameworks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Grpc.Tools" Version="__GRPC_NUGET_VERSION__" />
<!-- Grpc.Core.Api is not the package under test, it's included only so that gRPC generated service stubs can compile. -->
<PackageReference Include="Grpc.Core.Api" Version="2.46.5" />
<PackageReference Include="Google.Protobuf" Version="3.21.8" />
</ItemGroup>
<ItemGroup>
<Protobuf Include="**\*.proto" />
</ItemGroup>
<!-- Needed for the net45 build to work on Unix. See https://github.com/dotnet/designs/pull/33 -->
<ItemGroup>
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0" PrivateAssets="All" />
</ItemGroup>
</Project>

@ -0,0 +1,63 @@
#region Copyright notice and license
// Copyright 2015 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#endregion
using System;
using System.Linq;
using System.Threading.Tasks;
using Grpc.Core;
using Helloworld;
namespace TestGrpcPackage
{
class MainClass
{
public static void Main(string[] args)
{
CheckGreeterProtobufCodegenWorks();
CheckGreeterGrpcProtobufPluginCodegenWorks();
CheckDuplicateProtoFilesAreOk();
}
private static object CheckGreeterProtobufCodegenWorks()
{
return new HelloRequest { Name = "ABC" };
}
private static object CheckGreeterGrpcProtobufPluginCodegenWorks()
{
return typeof(GreeterImpl);
}
// Test that codegen works well in case the .csproj has .proto files
// of the same name, but under different directories (see #17672).
// This method doesn't need to be used, it is enough to check that it builds.
private static object CheckDuplicateProtoFilesAreOk()
{
return new DuplicateProto.MessageFromDuplicateProto();
}
}
class GreeterImpl : Greeter.GreeterBase
{
// Server side handler of the SayHello RPC
public override Task<HelloReply> SayHello(HelloRequest request, ServerCallContext context)
{
return Task.FromResult(new HelloReply { Message = "Hello " + request.Name });
}
}
}

@ -0,0 +1,23 @@
// Copyright 2019 The gRPC Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Test that codegen works well in case the .csproj has .proto files
// of the same name, but under different directories (see #17672).
syntax = "proto3";
package duplicate_proto;
message MessageFromDuplicateProto {
string name = 1;
}

@ -0,0 +1,29 @@
// Copyright 2019 The gRPC Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package helloworld;
service Greeter {
rpc SayHello (HelloRequest) returns (HelloReply) {}
}
message HelloRequest {
string name = 1;
}
message HelloReply {
string message = 1;
}

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<add key="https://www.nuget.org/api/v2/" value="https://www.nuget.org/api/v2/" />
<add key="TestNugetFeed" value="TestNugetFeed" />
</packageSources>
</configuration>

@ -0,0 +1,81 @@
#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
cd "$(dirname "$0")"
unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets_windows_dotnetcli.zip" -d TestNugetFeed
./update_version.sh auto
cd DistribTest
# TODO(jtattermusch): make sure we don't pollute the global nuget cache with
# the nugets being tested.
dotnet restore DistribTestDotNet.csproj
dotnet build DistribTestDotNet.csproj
ls -R bin
if [ "${SKIP_NET45_DISTRIBTEST}" != "1" ]
then
dotnet publish -f net45 DistribTestDotNet.csproj
# .NET 4.5 target after dotnet build
mono bin/Debug/net45/publish/DistribTestDotNet.exe
# .NET 4.5 target after dotnet publish
mono bin/Debug/net45/publish/DistribTestDotNet.exe
fi
if [ "${SKIP_NETCOREAPP21_DISTRIBTEST}" != "1" ]
then
dotnet publish -f netcoreapp2.1 DistribTestDotNet.csproj
# .NET Core target after dotnet build
dotnet exec bin/Debug/netcoreapp2.1/DistribTestDotNet.dll
# .NET Core target after dotnet publish
dotnet exec bin/Debug/netcoreapp2.1/publish/DistribTestDotNet.dll
fi
if [ "${SKIP_NETCOREAPP31_DISTRIBTEST}" != "1" ]
then
dotnet publish -f netcoreapp3.1 DistribTestDotNet.csproj
# .NET Core target after dotnet build
dotnet exec bin/Debug/netcoreapp3.1/DistribTestDotNet.dll
# .NET Core target after dotnet publish
dotnet exec bin/Debug/netcoreapp3.1/publish/DistribTestDotNet.dll
fi
if [ "${SKIP_NET50_DISTRIBTEST}" != "1" ]
then
dotnet publish -f net5.0 DistribTestDotNet.csproj
dotnet publish -r linux-x64 -f net5.0 DistribTestDotNet.csproj -p:PublishSingleFile=true --self-contained true --output net5_singlefile_publish
# .NET Core target after dotnet build
dotnet exec bin/Debug/net5.0/DistribTestDotNet.dll
# .NET Core target after dotnet publish
dotnet exec bin/Debug/net5.0/publish/DistribTestDotNet.dll
# binary generated by the single file publish
./net5_singlefile_publish/DistribTestDotNet
fi

@ -0,0 +1,31 @@
#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
cd "$(dirname "$0")"
CSHARP_VERSION="$1"
if [ "$CSHARP_VERSION" == "auto" ]
then
# autodetect C# version from the name of Grpc.Tools.0.0.0-x.nupkg file
# TODO: find a better shellcheck-compliant way to write the following line
# shellcheck disable=SC2010
CSHARP_VERSION=$(ls TestNugetFeed | grep -m 1 '^Grpc\.Tools\.[0-9].*\.nupkg$' | sed s/^Grpc\.Tools\.// | sed s/\.nupkg$// | sed s/\.symbols$//)
echo "Autodetected nuget ${CSHARP_VERSION}"
fi
# Replaces version placeholder with value provided as first argument.
sed -ibak "s/__GRPC_NUGET_VERSION__/${CSHARP_VERSION}/g" DistribTest/DistribTestDotNet.csproj

@ -31,9 +31,6 @@ source tools/internal_ci/helper_scripts/prepare_qemu_rc
# configure ccache
source tools/internal_ci/helper_scripts/prepare_ccache_rc
# Build all C# linux artifacts
tools/run_tests/task_runner.py -f artifact linux csharp ${TASK_RUNNER_EXTRA_FILTERS} -j 4 --inner_jobs 8 -x build_artifacts_csharp/sponge_log.xml || FAILED="true"
# Build all protoc linux artifacts
tools/run_tests/task_runner.py -f artifact linux protoc ${TASK_RUNNER_EXTRA_FILTERS} -j 4 --inner_jobs 8 -x build_artifacts_protoc/sponge_log.xml || FAILED="true"

@ -24,9 +24,6 @@ cd $(dirname $0)/../../..
export PREPARE_BUILD_INSTALL_DEPS_CSHARP=true
source tools/internal_ci/helper_scripts/prepare_build_macos_rc
# Build all C# macos artifacts
tools/run_tests/task_runner.py -f artifact macos csharp ${TASK_RUNNER_EXTRA_FILTERS} -j 2 --inner_jobs 4 -x build_artifacts_csharp/sponge_log.xml || FAILED="true"
# Build all protoc macos artifacts
tools/run_tests/task_runner.py -f artifact macos protoc ${TASK_RUNNER_EXTRA_FILTERS} -j 2 --inner_jobs 4 -x build_artifacts_protoc/sponge_log.xml || FAILED="true"

@ -38,9 +38,6 @@ curl -sSL --fail -o C:\zip\zip.exe https://storage.googleapis.com/grpc-build-hel
set PATH=C:\zip;%PATH%
zip --version
@rem Build all C# windows artifacts
python tools/run_tests/task_runner.py -f artifact windows csharp %TASK_RUNNER_EXTRA_FILTERS% -j 4 --inner_jobs 4 -x build_artifacts_csharp/sponge_log.xml || set FAILED=true
@rem Build all protoc windows artifacts
python tools/run_tests/task_runner.py -f artifact windows protoc %TASK_RUNNER_EXTRA_FILTERS% -j 4 --inner_jobs 4 -x build_artifacts_protoc/sponge_log.xml || set FAILED=true

@ -248,77 +248,6 @@ class RubyArtifact:
environ=environ)
class CSharpExtArtifact:
"""Builds C# native extension library"""
def __init__(self, platform, arch, arch_abi=None, presubmit=False):
self.name = 'csharp_ext_%s_%s' % (platform, arch)
self.platform = platform
self.arch = arch
self.arch_abi = arch_abi
self.labels = ['artifact', 'csharp', platform, arch]
if arch_abi:
self.name += '_%s' % arch_abi
self.labels.append(arch_abi)
if presubmit:
self.labels.append('presubmit')
def pre_build_jobspecs(self):
return []
def build_jobspec(self, inner_jobs=None):
environ = {}
if inner_jobs is not None:
# set number of parallel jobs when building native extension
environ['GRPC_CSHARP_BUILD_EXT_COMPILER_JOBS'] = str(inner_jobs)
if self.arch == 'android':
environ['ANDROID_ABI'] = self.arch_abi
return create_docker_jobspec(
self.name,
'tools/dockerfile/grpc_artifact_android_ndk',
'tools/run_tests/artifacts/build_artifact_csharp_android.sh',
environ=environ)
elif self.arch == 'ios':
return create_jobspec(
self.name,
['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'],
timeout_seconds=60 * 60,
use_workspace=True,
environ=environ)
elif self.platform == 'windows':
return create_jobspec(self.name, [
'tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
self.arch
],
timeout_seconds=45 * 60,
use_workspace=True,
environ=environ)
else:
if self.platform == 'linux':
dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format(
self.arch)
if self.arch == 'aarch64':
# for aarch64, use a dockcross manylinux image that will
# give us both ready to use crosscompiler and sufficient backward compatibility
dockerfile_dir = 'tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64'
return create_docker_jobspec(
self.name,
dockerfile_dir,
'tools/run_tests/artifacts/build_artifact_csharp.sh',
environ=environ)
else:
return create_jobspec(
self.name,
['tools/run_tests/artifacts/build_artifact_csharp.sh'],
timeout_seconds=45 * 60,
use_workspace=True,
environ=environ)
def __str__(self):
return self.name
class PHPArtifact:
"""Builds PHP PECL package"""
@ -423,21 +352,6 @@ def targets():
ProtocArtifact('macos', 'x64', presubmit=True),
ProtocArtifact('windows', 'x64', presubmit=True),
ProtocArtifact('windows', 'x86', presubmit=True),
CSharpExtArtifact('linux', 'x64', presubmit=True),
CSharpExtArtifact('linux', 'aarch64', presubmit=True),
CSharpExtArtifact('macos', 'x64', presubmit=True),
CSharpExtArtifact('windows', 'x64', presubmit=True),
CSharpExtArtifact('windows', 'x86', presubmit=True),
CSharpExtArtifact('linux',
'android',
arch_abi='arm64-v8a',
presubmit=True),
CSharpExtArtifact('linux',
'android',
arch_abi='armeabi-v7a',
presubmit=True),
CSharpExtArtifact('linux', 'android', arch_abi='x86', presubmit=True),
CSharpExtArtifact('macos', 'ios', presubmit=True),
PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m', presubmit=True),
PythonArtifact('manylinux2014', 'x64', 'cp38-cp38', presubmit=True),
PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'),

@ -1,16 +0,0 @@
@rem Copyright 2016 gRPC authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem Nothing to do here. C# has been removed from this repository. This script is a placeholder
@rem to prevent C# tests from becoming red (until they get eventually disabled).

@ -1,19 +0,0 @@
#!/bin/bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# Nothing to do here. C# has been removed from this repository. This script is a placeholder
# to prevent C# tests from becoming red (until they get eventually disabled).

@ -1,19 +0,0 @@
#!/bin/bash
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# Nothing to do here. C# has been removed from this repository. This script is a placeholder
# to prevent C# tests from becoming red (until they get eventually disabled).

@ -1,19 +0,0 @@
#!/bin/bash
# Copyright 2018 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# Nothing to do here. C# has been removed from this repository. This script is a placeholder
# to prevent C# tests from becoming red (until they get eventually disabled).

@ -115,28 +115,25 @@ class CSharpDistribTest(object):
self.name,
'tools/dockerfile/distribtest/csharp_%s_%s' %
(self.docker_suffix, self.arch),
'tools/run_tests/artifacts/run_distribtest_csharp.sh',
copy_rel_path='tools/run_tests/artifacts')
'test/distrib/csharp/run_distrib_test%s.sh' %
self.script_suffix,
copy_rel_path='test/distrib')
elif self.platform == 'macos':
return create_jobspec(
self.name,
['tools/run_tests/artifacts/run_distribtest_csharp.sh'],
environ={'EXTERNAL_GIT_ROOT': '../../../..'},
use_workspace=True)
return create_jobspec(self.name, [
'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix
],
environ={
'EXTERNAL_GIT_ROOT': '../../../..',
'SKIP_NETCOREAPP21_DISTRIBTEST': '1',
'SKIP_NET50_DISTRIBTEST': '1',
},
use_workspace=True)
elif self.platform == 'windows':
if self.arch == 'x64':
# Use double leading / as the first occurrence gets removed by msys bash
# when invoking the .bat file (side-effect of posix path conversion)
environ = {
'MSBUILD_EXTRA_ARGS': '//p:Platform=x64',
'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'
}
else:
environ = {'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'}
# TODO(jtattermusch): re-enable windows distribtest
return create_jobspec(
self.name,
['bash', 'tools/run_tests/artifacts/run_distribtest_csharp.sh'],
environ=environ,
environ={},
use_workspace=True)
else:
raise Exception("Not supported yet.")
@ -393,14 +390,11 @@ def targets():
testcase='cmake_as_externalproject',
presubmit=True),
# C#
CSharpDistribTest('linux', 'x64', 'stretch', presubmit=True),
CSharpDistribTest('linux',
'x64',
'stretch',
use_dotnet_cli=True,
presubmit=True),
CSharpDistribTest('linux', 'x64', 'centos7'),
CSharpDistribTest('linux', 'x64', 'ubuntu1604'),
CSharpDistribTest('linux', 'x64', 'ubuntu1604', use_dotnet_cli=True),
CSharpDistribTest('linux',
'x64',
@ -417,7 +411,7 @@ def targets():
'dotnet5',
use_dotnet_cli=True,
presubmit=True),
CSharpDistribTest('macos', 'x64', presubmit=True),
CSharpDistribTest('macos', 'x64', use_dotnet_cli=True, presubmit=True),
CSharpDistribTest('windows', 'x86', presubmit=True),
CSharpDistribTest('windows', 'x64', presubmit=True),
# Python

Loading…
Cancel
Save