From a93e3d275335f3a95f9d2a69da26281d282d46f9 Mon Sep 17 00:00:00 2001 From: kkm Date: Sun, 4 Feb 2018 17:50:39 -0800 Subject: [PATCH 01/53] Add Grpc.Tools MsBuild taks assembly, test and scripting This is a complete set of tooling to build .proto files, with or without gRPC services, in .csproj, both "classic" and SDK flavors, and a bare minimum support for C++ projects. Highlights and omissions: * By default, generated files are placed into project's intermediate directory under obj/, and treated as temporary generated sources. * The projects are highly customizabe thorugh item metadata on Protobuf items. * SDK projects only use Visual Studio new build system, and automatically import XAML property sheets that allow setting per-file properties, such as generated file access, and whether to expect gRPC outputs, from VS properties windows. This possibly requires VS restart after the package is added to solution. Classic projects cannot be extended this way, and only show Protobuf as the possible item; settings are modified by editing the project only. * For C++ projects, only the tool and standard proto import paths are provided, no custom targets yet. This is in the works. * gRPC and Protobuf scripts are separate, and everything is programmed to easily split the Tools package into one for Google.Protobuf and another for Grpc.Tools. This requires tighter coordination between the teams. * The tasks DLL knows about gRPC. I tried to use it to support gRPC in a script-only fashion, but using the tasks results in much cleaner scripts. This is probably how it should remain. * In multitarget projects (multiple frameworks) protoc files are compiled for each target, and also for Debug/Release configuration sepatately. A possible fix is in the works, but requries some MsBuild tooling fixes, so it will take a while. * There are 4 tasks. The "smart" task predicts protoc outputs, and knows things about protoc naming conventions. This supports only C# and C++. The "dumb" task simply invokes protoc in a language-independent way, and supports all languages known to protoc. In the (not very likely) case protoc is used with MsBuild for these languages, instructions for extending the build is provided in build script comments. The other 2 tasks are one to detect current platform and therefore tools paths, and another to read protoc generated dependency file. We use it for C#, but custom project may opt not to use the dependecy files. * 64-bit tools for Windows (protoc and grpc plugin exe) have been removed from package, as Windows is alsways able to run 32-bit executable (and they are smaller and faster, and always preferred when 2G address space is enough). --- .../Grpc.Tools.Tests/DepFileUtilTests.cs | 134 ++++++ src/csharp/Grpc.Tools.Tests/GeneratorTests.cs | 165 +++++++ .../Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 27 ++ src/csharp/Grpc.Tools.Tests/NUnitMain.cs | 31 ++ .../Grpc.Tools.Tests/ProtoCompileTaskTest.cs | 232 ++++++++++ .../ProtoToolsPlatformTaskTest.cs | 53 +++ src/csharp/Grpc.Tools.Tests/Utils.cs | 43 ++ src/csharp/Grpc.Tools.nuspec | 33 -- src/csharp/Grpc.Tools/Common.cs | 105 +++++ src/csharp/Grpc.Tools/DepFileUtil.cs | 198 +++++++++ src/csharp/Grpc.Tools/GeneratorServices.cs | 168 +++++++ src/csharp/Grpc.Tools/Grpc.Tools.csproj | 95 ++++ src/csharp/Grpc.Tools/ProtoCompile.cs | 409 ++++++++++++++++++ src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs | 80 ++++ .../Grpc.Tools/ProtoReadDependencies.cs | 70 +++ src/csharp/Grpc.Tools/ProtoToolsPlatform.cs | 58 +++ src/csharp/Grpc.Tools/build/Grpc.Tools.props | 11 + .../Grpc.Tools/build/Grpc.Tools.targets | 11 + .../Grpc.Tools/build/_grpc/Grpc.CSharp.xml | 30 ++ src/csharp/Grpc.Tools/build/_grpc/README | 3 + .../Grpc.Tools/build/_grpc/_Grpc.Tools.props | 6 + .../build/_grpc/_Grpc.Tools.targets | 46 ++ .../_protobuf/Google.Protobuf.Tools.props | 23 + .../_protobuf/Google.Protobuf.Tools.targets | 383 ++++++++++++++++ .../build/_protobuf/Protobuf.CSharp.xml | 99 +++++ src/csharp/Grpc.Tools/build/_protobuf/README | 1 + .../Grpc.Tools/build/native/Grpc.Tools.props | 15 + src/csharp/Grpc.sln | 12 + src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 2 +- src/csharp/tests.json | 10 + .../build_packages_dotnetcli.bat.template | 2 +- .../build_packages_dotnetcli.sh.template | 2 +- 33 files changed, 2522 insertions(+), 37 deletions(-) create mode 100644 src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs create mode 100644 src/csharp/Grpc.Tools.Tests/GeneratorTests.cs create mode 100644 src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj create mode 100644 src/csharp/Grpc.Tools.Tests/NUnitMain.cs create mode 100644 src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs create mode 100644 src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs create mode 100644 src/csharp/Grpc.Tools.Tests/Utils.cs delete mode 100644 src/csharp/Grpc.Tools.nuspec create mode 100644 src/csharp/Grpc.Tools/Common.cs create mode 100644 src/csharp/Grpc.Tools/DepFileUtil.cs create mode 100644 src/csharp/Grpc.Tools/GeneratorServices.cs create mode 100644 src/csharp/Grpc.Tools/Grpc.Tools.csproj create mode 100644 src/csharp/Grpc.Tools/ProtoCompile.cs create mode 100644 src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs create mode 100644 src/csharp/Grpc.Tools/ProtoReadDependencies.cs create mode 100644 src/csharp/Grpc.Tools/ProtoToolsPlatform.cs create mode 100644 src/csharp/Grpc.Tools/build/Grpc.Tools.props create mode 100644 src/csharp/Grpc.Tools/build/Grpc.Tools.targets create mode 100644 src/csharp/Grpc.Tools/build/_grpc/Grpc.CSharp.xml create mode 100644 src/csharp/Grpc.Tools/build/_grpc/README create mode 100644 src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.props create mode 100644 src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets create mode 100644 src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props create mode 100644 src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets create mode 100644 src/csharp/Grpc.Tools/build/_protobuf/Protobuf.CSharp.xml create mode 100644 src/csharp/Grpc.Tools/build/_protobuf/README create mode 100644 src/csharp/Grpc.Tools/build/native/Grpc.Tools.props diff --git a/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs b/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs new file mode 100644 index 00000000000..0ea621adea5 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs @@ -0,0 +1,134 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.IO; +using Grpc.Tools; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NUnit.Framework; + +namespace Grps.Tools.Tests { + public class DepFileUtilTests { + + [Test] + public void HashString64Hex_IsSane() { + string hashFoo1 = DepFileUtil.HashString64Hex("foo"); + string hashEmpty = DepFileUtil.HashString64Hex(""); + string hashFoo2 = DepFileUtil.HashString64Hex("foo"); + + StringAssert.IsMatch("^[a-f0-9]{16}$", hashFoo1); + Assert.AreEqual(hashFoo1, hashFoo2); + Assert.AreNotEqual(hashFoo1, hashEmpty); + } + + [Test] + public void GetDepFilenameForProto_IsSane() { + StringAssert.IsMatch(@"^out[\\/][a-f0-9]{16}_foo.protodep$", + DepFileUtil.GetDepFilenameForProto("out", "foo.proto")); + StringAssert.IsMatch(@"^[a-f0-9]{16}_foo.protodep$", + DepFileUtil.GetDepFilenameForProto("", "foo.proto")); + } + + [Test] + public void GetDepFilenameForProto_HashesDir() { + string PickHash(string fname) => + DepFileUtil.GetDepFilenameForProto("", fname).Substring(0, 16); + + string same1 = PickHash("dir1/dir2/foo.proto"); + string same2 = PickHash("dir1/dir2/proto.foo"); + string same3 = PickHash("dir1/dir2/proto"); + string same4 = PickHash("dir1/dir2/.proto"); + string unsame1 = PickHash("dir2/foo.proto"); + string unsame2 = PickHash("/dir2/foo.proto"); + + Assert.AreEqual(same1, same2); + Assert.AreEqual(same1, same3); + Assert.AreEqual(same1, same4); + Assert.AreNotEqual(same1, unsame1); + Assert.AreNotEqual(unsame1, unsame2); + } + + ////////////////////////////////////////////////////////////////////////// + // Full file reading tests + + // Generated by protoc on Windows. Slashes vary. + const string depFile1 = +@"C:\projects\foo\src\./foo.grpc.pb.cc \ +C:\projects\foo\src\./foo.grpc.pb.h \ +C:\projects\foo\src\./foo.pb.cc \ + C:\projects\foo\src\./foo.pb.h: C:/usr/include/google/protobuf/wrappers.proto\ + C:/usr/include/google/protobuf/any.proto\ +C:/usr/include/google/protobuf/source_context.proto\ + C:/usr/include/google/protobuf/type.proto\ + foo.proto"; + + // This has a nasty output directory with a space. + const string depFile2 = +@"obj\Release x64\net45\/Foo.cs \ +obj\Release x64\net45\/FooGrpc.cs: C:/usr/include/google/protobuf/wrappers.proto\ + C:/projects/foo/src//foo.proto"; + + [Test] + public void ReadDependencyInput_FullFile1() { + string[] deps = ReadDependencyInputFromFileData(depFile1, "foo.proto"); + + Assert.NotNull(deps); + Assert.That(deps, Has.Length.InRange(4, 5)); // foo.proto may or may not be listed. + Assert.That(deps, Has.One.EndsWith("wrappers.proto")); + Assert.That(deps, Has.One.EndsWith("type.proto")); + Assert.That(deps, Has.None.StartWith(" ")); + } + + [Test] + public void ReadDependencyInput_FullFile2() { + string[] deps = ReadDependencyInputFromFileData(depFile2, "C:/projects/foo/src/foo.proto"); + + Assert.NotNull(deps); + Assert.That(deps, Has.Length.InRange(1, 2)); + Assert.That(deps, Has.One.EndsWith("wrappers.proto")); + Assert.That(deps, Has.None.StartWith(" ")); + } + + [Test] + public void ReadDependencyInput_FullFileUnparsable() { + string[] deps = ReadDependencyInputFromFileData("a:/foo.proto", "/foo.proto"); + Assert.NotNull(deps); + Assert.Zero(deps.Length); + } + + // NB in our tests files are put into the temp directory but all have + // different names. Avoid adding files with the same directory path and + // name, or add reasonable handling for it if required. Tests are run in + // parallel and will collide otherwise. + private string[] ReadDependencyInputFromFileData(string fileData, string protoName) { + string tempPath = Path.GetTempPath(); + string tempfile = DepFileUtil.GetDepFilenameForProto(tempPath, protoName); + try { + File.WriteAllText(tempfile, fileData); + var mockEng = new Moq.Mock(); + var log = new TaskLoggingHelper(mockEng.Object, "x"); + return DepFileUtil.ReadDependencyInputs(tempPath, protoName, log); + } finally { + try { + File.Delete(tempfile); + } catch { } + } + } + } +} diff --git a/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs new file mode 100644 index 00000000000..0a273380b98 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs @@ -0,0 +1,165 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + public class GeneratorTests { + protected Mock _mockEngine; + protected TaskLoggingHelper _log; + + [SetUp] + public void SetUp() { + _mockEngine = new Mock(); + _log = new TaskLoggingHelper(_mockEngine.Object, "dummy"); + } + + [TestCase("csharp")] + [TestCase("CSharp")] + [TestCase("cpp")] + public void ValidLanguages(string lang) { + Assert.IsNotNull(GeneratorServices.GetForLanguage(lang, _log)); + _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Never); + } + + [TestCase("")] + [TestCase("COBOL")] + public void InvalidLanguages(string lang) { + Assert.IsNull(GeneratorServices.GetForLanguage(lang, _log)); + _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Once); + } + }; + + public class CSharpGeneratorTests : GeneratorTests { + GeneratorServices _generator; + + [SetUp] + public new void SetUp() { + _generator = GeneratorServices.GetForLanguage("CSharp", _log); + } + + [TestCase("foo.proto", "Foo.cs", "FooGrpc.cs")] + [TestCase("sub/foo.proto", "Foo.cs", "FooGrpc.cs")] + [TestCase("one_two.proto", "OneTwo.cs", "OneTwoGrpc.cs")] + [TestCase("__one_two!.proto", "OneTwo!.cs", "OneTwo!Grpc.cs")] + [TestCase("one(two).proto", "One(two).cs", "One(two)Grpc.cs")] + [TestCase("one_(two).proto", "One(two).cs", "One(two)Grpc.cs")] + [TestCase("one two.proto", "One two.cs", "One twoGrpc.cs")] + [TestCase("one_ two.proto", "One two.cs", "One twoGrpc.cs")] + [TestCase("one .proto", "One .cs", "One Grpc.cs")] + public void NameMangling(string proto, string expectCs, string expectGrpcCs) { + var poss = _generator.GetPossibleOutputs(Utils.MakeItem(proto, "grpcservices", "both")); + Assert.AreEqual(2, poss.Length); + Assert.Contains(expectCs, poss); + Assert.Contains(expectGrpcCs, poss); + } + + [Test] + public void NoGrpcOneOutput() { + var poss = _generator.GetPossibleOutputs(Utils.MakeItem("foo.proto")); + Assert.AreEqual(1, poss.Length); + } + + [TestCase("none")] + [TestCase("")] + public void GrpcNoneOneOutput(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(1, poss.Length); + } + + [TestCase("client")] + [TestCase("server")] + [TestCase("both")] + public void GrpcEnabledTwoOutputs(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(2, poss.Length); + } + + [Test] + public void OutputDirMetadataRecognized() { + var item = Utils.MakeItem("foo.proto", "OutputDir", "out"); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(1, poss.Length); + Assert.That(poss[0], Is.EqualTo("out/Foo.cs") | Is.EqualTo("out\\Foo.cs")); + } + }; + + public class CppGeneratorTests : GeneratorTests { + GeneratorServices _generator; + + [SetUp] + public new void SetUp() { + _generator = GeneratorServices.GetForLanguage("Cpp", _log); + } + + [TestCase("foo.proto", "", "foo")] + [TestCase("foo.proto", ".", "foo")] + [TestCase("foo.proto", "./", "foo")] + [TestCase("sub/foo.proto", "", "sub/foo")] + [TestCase("root/sub/foo.proto", "root", "sub/foo")] + [TestCase("root/sub/foo.proto", "root", "sub/foo")] + [TestCase("/root/sub/foo.proto", "/root", "sub/foo")] + public void RelativeDirectoryCompute(string proto, string root, string expectStem) { + if (Path.DirectorySeparatorChar == '\\') + expectStem = expectStem.Replace('/', '\\'); + var poss = _generator.GetPossibleOutputs(Utils.MakeItem(proto, "ProtoRoot", root)); + Assert.AreEqual(2, poss.Length); + Assert.Contains(expectStem + ".pb.cc", poss); + Assert.Contains(expectStem + ".pb.h", poss); + } + + [Test] + public void NoGrpcTwoOutputs() { + var poss = _generator.GetPossibleOutputs(Utils.MakeItem("foo.proto")); + Assert.AreEqual(2, poss.Length); + } + + [TestCase("false")] + [TestCase("")] + public void GrpcDisabledTwoOutput(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(2, poss.Length); + } + + [TestCase("true")] + public void GrpcEnabledFourOutputs(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(4, poss.Length); + Assert.Contains("foo.pb.cc", poss); + Assert.Contains("foo.pb.h", poss); + Assert.Contains("foo_grpc.pb.cc", poss); + Assert.Contains("foo_grpc.pb.h", poss); + } + + [Test] + public void OutputDirMetadataRecognized() { + var item = Utils.MakeItem("foo.proto", "OutputDir", "out"); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(2, poss.Length); + Assert.That(Path.GetDirectoryName(poss[0]), Is.EqualTo("out")); + } + }; +} diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj new file mode 100644 index 00000000000..585e4518b59 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp1.0;net45 + Exe + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs new file mode 100644 index 00000000000..c4452c50d2c --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs @@ -0,0 +1,31 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Reflection; +using NUnitLite; + +namespace Grps.Tools.Tests { + static class NUnitMain { + public static int Main(string[] args) => +#if NETCOREAPP1_0 + new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args); +#else + new AutoRun().Execute(args); +#endif + }; +} diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs new file mode 100644 index 00000000000..86c78289b29 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs @@ -0,0 +1,232 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Reflection; +using Microsoft.Build.Framework; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + public class ProtoCompileBasicTests { + // Mock task class that stops right before invoking protoc. + public class ProtoCompileTestable : ProtoCompile { + public string LastPathToTool { get; private set; } + public string[] LastResponseFile { get; private set; } + + protected override int ExecuteTool(string pathToTool, + string response, + string commandLine) { + // We should never be using command line commands. + Assert.That(commandLine, Is.Null | Is.Empty); + + // Must receive a path to tool + Assert.That(pathToTool, Is.Not.Null & Is.Not.Empty); + Assert.That(response, Is.Not.Null & Does.EndWith("\n")); + + LastPathToTool = pathToTool; + LastResponseFile = response.Remove(response.Length - 1).Split('\n'); + + // Do not run the tool, but pretend it ran successfully. + return 0; + } + }; + + protected Mock _mockEngine; + protected ProtoCompileTestable _task; + + [SetUp] + public void SetUp() { + _mockEngine = new Mock(); + _task = new ProtoCompileTestable { + BuildEngine = _mockEngine.Object + }; + } + + [TestCase("ProtoBuf")] + [TestCase("Generator")] + [TestCase("OutputDir")] + [Description("We trust MSBuild to initialize these properties.")] + public void RequiredAttributePresentOnProperty(string prop) { + var pinfo = _task.GetType()?.GetProperty(prop); + Assert.NotNull(pinfo); + Assert.That(pinfo, Has.Attribute()); + } + }; + + internal class ProtoCompileCommandLineGeneratorTests : ProtoCompileBasicTests { + [SetUp] + public new void SetUp() { + _task.Generator = "csharp"; + _task.OutputDir = "outdir"; + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); + } + + void ExecuteExpectSuccess() { + _mockEngine + .Setup(me => me.LogErrorEvent(It.IsAny())) + .Callback((BuildErrorEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")); + bool result = _task.Execute(); + Assert.IsTrue(result); + } + + [Test] + public void MinimalCompile() { + ExecuteExpectSuccess(); + Assert.That(_task.LastPathToTool, Does.Match(@"protoc(.exe)?$")); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + "--csharp_out=outdir", "a.proto" })); + } + + [Test] + public void CompileTwoFiles() { + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto", "foo/b.proto"); + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + "--csharp_out=outdir", "a.proto", "foo/b.proto" })); + } + + [Test] + public void CompileWithProtoPaths() { + _task.ProtoPath = new[] { "/path1", "/path2" }; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + "--csharp_out=outdir", "--proto_path=/path1", + "--proto_path=/path2", "a.proto" })); + } + + [TestCase("Cpp")] + [TestCase("CSharp")] + [TestCase("Java")] + [TestCase("Javanano")] + [TestCase("Js")] + [TestCase("Objc")] + [TestCase("Php")] + [TestCase("Python")] + [TestCase("Ruby")] + public void CompileWithOptions(string gen) { + _task.Generator = gen; + _task.OutputOptions = new[] { "foo", "bar" }; + ExecuteExpectSuccess(); + gen = gen.ToLowerInvariant(); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + $"--{gen}_out=outdir", $"--{gen}_opt=foo,bar", "a.proto" })); + } + + [Test] + public void OutputDependencyFile() { + _task.DependencyOut = "foo/my.protodep"; + // Task fails trying to read the non-generated file; we ignore that. + _task.Execute(); + Assert.That(_task.LastResponseFile, + Does.Contain("--dependency_out=foo/my.protodep")); + } + + [Test] + public void OutputDependencyWithProtoDepDir() { + _task.ProtoDepDir = "foo"; + // Task fails trying to read the non-generated file; we ignore that. + _task.Execute(); + Assert.That(_task.LastResponseFile, + Has.One.Match(@"^--dependency_out=foo[/\\].+_a.protodep$")); + } + + [Test] + public void GenerateGrpc() { + _task.GrpcPluginExe = "/foo/grpcgen"; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { + "--csharp_out=outdir", "--grpc_out=outdir", + "--plugin=protoc-gen-grpc=/foo/grpcgen" })); + } + + [Test] + public void GenerateGrpcWithOutDir() { + _task.GrpcPluginExe = "/foo/grpcgen"; + _task.GrpcOutputDir = "gen-out"; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { + "--csharp_out=outdir", "--grpc_out=gen-out" })); + } + + [Test] + public void GenerateGrpcWithOptions() { + _task.GrpcPluginExe = "/foo/grpcgen"; + _task.GrpcOutputOptions = new[] { "baz", "quux" }; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, + Does.Contain("--grpc_opt=baz,quux")); + } + + [Test] + public void DirectoryArgumentsSlashTrimmed() { + _task.GrpcPluginExe = "/foo/grpcgen"; + _task.GrpcOutputDir = "gen-out/"; + _task.OutputDir = "outdir/"; + _task.ProtoPath = new[] { "/path1/", "/path2/" }; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { + "--proto_path=/path1", "--proto_path=/path2", + "--csharp_out=outdir", "--grpc_out=gen-out" })); + } + + [TestCase("." , ".")] + [TestCase("/" , "/")] + [TestCase("//" , "/")] + [TestCase("/foo/" , "/foo")] + [TestCase("/foo" , "/foo")] + [TestCase("foo/" , "foo")] + [TestCase("foo//" , "foo")] + [TestCase("foo/\\" , "foo")] + [TestCase("foo\\/" , "foo")] + [TestCase("C:\\foo", "C:\\foo")] + [TestCase("C:" , "C:")] + [TestCase("C:\\" , "C:\\")] + [TestCase("C:\\\\" , "C:\\")] + public void DirectorySlashTrimmingCases(string given, string expect) { + _task.OutputDir = given; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, + Does.Contain("--csharp_out=" + expect)); + } + }; + + internal class ProtoCompileCommandLinePrinterTests : ProtoCompileBasicTests { + [SetUp] + public new void SetUp() { + _task.Generator = "csharp"; + _task.OutputDir = "outdir"; + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); + + _mockEngine + .Setup(me => me.LogMessageEvent(It.IsAny())) + .Callback((BuildMessageEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")) + .Verifiable("Command line was not output by the task."); + } + + void ExecuteExpectSuccess() { + _mockEngine + .Setup(me => me.LogErrorEvent(It.IsAny())) + .Callback((BuildErrorEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")); + bool result = _task.Execute(); + Assert.IsTrue(result); + } + }; +} diff --git a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs new file mode 100644 index 00000000000..3ba0bdfbf6b --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs @@ -0,0 +1,53 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using Microsoft.Build.Framework; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + // This test requires that environment variables be set to the exected + // output of the task in its external test harness: + // PROTOTOOLS_TEST_CPU = { x64 | x86 } + // PROTOTOOLS_TEST_OS = { linux | macosx | windows } + public class ProtoToolsPlatformTaskTests { + static string s_expectOs; + static string s_expectCpu; + + [OneTimeSetUp] + public static void Init() { + s_expectCpu = Environment.GetEnvironmentVariable("PROTOTOOLS_TEST_CPU"); + s_expectOs = Environment.GetEnvironmentVariable("PROTOTOOLS_TEST_OS"); + if (s_expectCpu == null || s_expectOs == null) + Assert.Inconclusive("This test requires PROTOTOOLS_TEST_CPU and " + + "PROTOTOOLS_TEST_OS set in the environment to match the OS it runs on."); + } + + [Test] + public void CpuAndOsMatchExpected() { + var mockEng = new Mock(); + var task = new ProtoToolsPlatform() { + BuildEngine = mockEng.Object + }; + task.Execute(); + Assert.AreEqual(s_expectCpu, task.Cpu); + Assert.AreEqual(s_expectOs, task.Os); + } + }; +} diff --git a/src/csharp/Grpc.Tools.Tests/Utils.cs b/src/csharp/Grpc.Tools.Tests/Utils.cs new file mode 100644 index 00000000000..618e3354524 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/Utils.cs @@ -0,0 +1,43 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools.Tests { + static class Utils { + // Build an item with a name from args[0] and metadata key-value pairs + // from the rest of args, interleaved. + // This does not do any checking, and expects an odd number of args. + public static ITaskItem MakeItem(params string[] args) { + var item = new TaskItem(args[0]); + for (int i = 1; i < args.Length; i += 2) + item.SetMetadata(args[i], args[i + 1]); + return item; + } + + // Return an array of items from given itemspecs. + public static ITaskItem[] MakeSimpleItems(params string[] specs) { + return specs.Select(s => new TaskItem(s)).ToArray(); + } + }; +} diff --git a/src/csharp/Grpc.Tools.nuspec b/src/csharp/Grpc.Tools.nuspec deleted file mode 100644 index 0cae5572fd8..00000000000 --- a/src/csharp/Grpc.Tools.nuspec +++ /dev/null @@ -1,33 +0,0 @@ - - - - Grpc.Tools - gRPC C# Tools - Tools for C# implementation of gRPC - an RPC library and framework - Precompiled protobuf compiler and gRPC protobuf compiler plugin for generating gRPC client/server C# code. Binaries are available for Windows, Linux and MacOS. - $version$ - Google Inc. - grpc-packages - https://github.com/grpc/grpc/blob/master/LICENSE - https://github.com/grpc/grpc - false - Release $version$ - Copyright 2015, Google Inc. - gRPC RPC Protocol HTTP/2 - - - - - - - - - - - - - - - - - diff --git a/src/csharp/Grpc.Tools/Common.cs b/src/csharp/Grpc.Tools/Common.cs new file mode 100644 index 00000000000..df539f8c4f9 --- /dev/null +++ b/src/csharp/Grpc.Tools/Common.cs @@ -0,0 +1,105 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security; + +[assembly: InternalsVisibleTo("Grpc.Tools.Tests")] + +namespace Grpc.Tools { + // Metadata names that we refer to often. + static class Metadata { + // On output dependency lists. + public static string kSource = "Source"; + // On ProtoBuf items. + public static string kProtoRoot = "ProtoRoot"; + public static string kOutputDir = "OutputDir"; + public static string kGrpcServices = "GrpcServices"; + public static string kGrpcOutputDir = "GrpcOutputDir"; + }; + + // A few flags used to control the behavior under various platforms. + internal static class Platform { + public enum OsKind { Unknown, Windows, Linux, MacOsX }; + public static readonly OsKind Os; + + public enum CpuKind { Unknown, X86, X64 }; + public static readonly CpuKind Cpu; + + // This is not necessarily true, but good enough. BCL lacks a per-FS + // API to determine file case sensitivity. + public static bool IsFsCaseInsensitive => Os == OsKind.Windows; + public static bool IsWindows => Os == OsKind.Windows; + + static Platform() { +#if NETSTANDARD + Os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? OsKind.Windows + : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? OsKind.Linux + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? OsKind.MacOsX + : OsKind.Unknown; + + switch (RuntimeInformation.OSArchitecture) { + case Architecture.X86: Cpu = CpuKind.X86; break; + case Architecture.X64: Cpu = CpuKind.X64; break; + // We do not have build tools for other architectures. + default: Cpu = CpuKind.Unknown; break; + } +#else + // Running under either Mono or full MS framework. + Os = OsKind.Windows; + if (Type.GetType("Mono.Runtime", throwOnError: false) != null) { + // Congratulations. We are running under Mono. + var plat = Environment.OSVersion.Platform; + if (plat == PlatformID.MacOSX) { + Os = OsKind.MacOsX; + } else if (plat == PlatformID.Unix || (int)plat == 128) { + // TODO(kkm): This is how Mono detects OSX internally. Looks cheesy + // to me. Would not testing for /proc absence be more reliable? OSX + // did never have it, AFAIK. + Os = File.Exists("/usr/lib/libc.dylib") ? OsKind.MacOsX : OsKind.Linux; + } + } + + // Hope we are not building on ARM under Xamarin! + Cpu = Environment.Is64BitOperatingSystem ? CpuKind.X64 : CpuKind.X86; +#endif + } + }; + + // Exception handling helpers. + static class Exceptions { + // Returns true iff the exception indicates an error from an I/O call. See + // https://github.com/Microsoft/msbuild/blob/v15.4.8.50001/src/Shared/ExceptionHandling.cs#L101 + static public bool IsIoRelated(Exception ex) => + ex is IOException || + (ex is ArgumentException && !(ex is ArgumentNullException)) || + ex is SecurityException || + ex is UnauthorizedAccessException || + ex is NotSupportedException; + }; + + // String helpers. + static class Strings { + // Compare string to argument using OrdinalIgnoreCase comparison. + public static bool EqualNoCase(this string a, string b) => + string.Equals(a, b, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/csharp/Grpc.Tools/DepFileUtil.cs b/src/csharp/Grpc.Tools/DepFileUtil.cs new file mode 100644 index 00000000000..2a931b7295d --- /dev/null +++ b/src/csharp/Grpc.Tools/DepFileUtil.cs @@ -0,0 +1,198 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools { + internal static class DepFileUtil { +/* + Sample dependency files. Notable features we have to deal with: + * Slash doubling, must normalize them. + * Spaces in file names. Cannot just "unwrap" the line on backslash at eof; + rather, treat every line as containing one file name except for one with + the ':' separator, as containing exactly two. + * Deal with ':' also being drive letter separator (second example). + +obj\Release\net45\/Foo.cs \ +obj\Release\net45\/FooGrpc.cs: C:/foo/include/google/protobuf/wrappers.proto\ + C:/projects/foo/src//foo.proto + +C:\projects\foo\src\./foo.grpc.pb.cc \ +C:\projects\foo\src\./foo.grpc.pb.h \ +C:\projects\foo\src\./foo.pb.cc \ +C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ + C:/foo/include/google/protobuf/any.proto\ + C:/foo/include/google/protobuf/source_context.proto\ + C:/foo/include/google/protobuf/type.proto\ + foo.proto +*/ + + // Read file names from the dependency file to the right of ':'. + public static string[] ReadDependencyInputs(string protoDepDir, string proto, + TaskLoggingHelper log) { + string depFilename = GetDepFilenameForProto(protoDepDir, proto); + string[] lines = ReadDepFileLines(depFilename, false, log); + if (lines.Length == 0) { + return lines; + } + + var result = new List(); + bool skip = true; + foreach (string line in lines) { + // Start at the only line separating dependency outputs from inputs. + int ix = skip ? FindLineSeparator(line) : -1; + skip = skip && ix < 0; + if (skip) continue; + string file = ExtractFilenameFromLine(line, ix + 1, line.Length); + if (file == "") { + log.LogMessage(MessageImportance.Low, + $"Skipping unparsable dependency file {depFilename}.\nLine with error: '{line}'"); + return new string[0]; + } + + // Do not bend over backwards trying not to include a proto into its + // own list of dependencies. Since a file is not older than self, + // it is safe to add; this is purely a memory optimization. + if (file != proto) { + result.Add(file); + } + } + return result.ToArray(); + } + + // Read file names from the dependency file to the left of ':'. + public static string[] ReadDependencyOutputs(string depFilename, + TaskLoggingHelper log) { + string[] lines = ReadDepFileLines(depFilename, true, log); + if (lines.Length == 0) { + return lines; + } + + var result = new List(); + foreach (string line in lines) { + int ix = FindLineSeparator(line); + string file = ExtractFilenameFromLine(line, 0, ix >= 0 ? ix : line.Length); + if (file == "") { + log.LogError("Unable to parse generated dependency file {0}.\n" + + "Line with error: '{1}'", depFilename, line); + return new string[0]; + } + result.Add(file); + + // If this is the line with the separator, do not read further. + if (ix >= 0) + break; + } + return result.ToArray(); + } + + // Get complete dependency file name from directory hash and file name, + // tucked onto protoDepDir, e. g. + // ("out", "foo/file.proto") => "out/deadbeef12345678_file.protodep". + // This way, the filenames are unique but still possible to make sense of. + public static string GetDepFilenameForProto(string protoDepDir, string proto) { + string dirname = Path.GetDirectoryName(proto); + if (Platform.IsFsCaseInsensitive) { + dirname = dirname.ToLowerInvariant(); + } + string dirhash = HashString64Hex(dirname); + string filename = Path.GetFileNameWithoutExtension(proto); + return Path.Combine(protoDepDir, $"{dirhash}_{filename}.protodep"); + } + + // Get a 64-bit hash for a directory string. We treat it as if it were + // unique, since there are not so many distinct proto paths in a project. + // We take the first 64 bit of the string SHA1. + // Internal for tests access only. + internal static string HashString64Hex(string str) { + using (var sha1 = System.Security.Cryptography.SHA1.Create()) { + byte[] hash = sha1.ComputeHash(Encoding.UTF8.GetBytes(str)); + var hashstr = new StringBuilder(16); + for (int i = 0; i < 8; i++) { + hashstr.Append(hash[i].ToString("x2")); + } + return hashstr.ToString(); + } + } + + // Extract filename between 'beg' (inclusive) and 'end' (exclusive) from + // line 'line', skipping over trailing and leading whitespace, and, when + // 'end' is immediately past end of line 'line', also final '\' (used + // as a line continuation token in the dep file). + // Returns an empty string if the filename cannot be extracted. + static string ExtractFilenameFromLine(string line, int beg, int end) { + while (beg < end && char.IsWhiteSpace(line[beg])) beg++; + if (beg < end && end == line.Length && line[end - 1] == '\\') end--; + while (beg < end && char.IsWhiteSpace(line[end - 1])) end--; + if (beg == end) return ""; + + string filename = line.Substring(beg, end - beg); + try { + // Normalize file name. + return Path.Combine( + Path.GetDirectoryName(filename), + Path.GetFileName(filename)); + } catch (Exception ex) when (Exceptions.IsIoRelated(ex)) { + return ""; + } + } + + // Finds the index of the ':' separating dependency clauses in the line, + // not taking Windows drive spec into account. Returns the index of the + // separating ':', or -1 if no separator found. + static int FindLineSeparator(string line) { + // Mind this case where the first ':' is not separator: + // C:\foo\bar\.pb.h: C:/protobuf/wrappers.proto\ + int ix = line.IndexOf(':'); + if (ix <= 0 || ix == line.Length - 1 + || (line[ix + 1] != '/' && line[ix + 1] != '\\') + || !char.IsLetter(line[ix - 1])) + return ix; // Not a windows drive: no letter before ':', or no '\' after. + for (int j = ix - 1; --j >= 0;) { + if (!char.IsWhiteSpace(line[j])) + return ix; // Not space or BOL only before "X:/". + } + return line.IndexOf(':', ix + 1); + } + + // Read entire dependency file. The 'required' parameter controls error + // logging behavior in case the file not found. We require this file when + // compiling, but reading it is optional when computing depnedencies. + static string[] ReadDepFileLines(string filename, bool required, + TaskLoggingHelper log) { + try { + var result = File.ReadAllLines(filename); + if (!required) + log.LogMessage(MessageImportance.Low, $"Using dependency file {filename}"); + return result; + } catch (Exception ex) when (Exceptions.IsIoRelated(ex)) { + if (required) { + log.LogError($"Unable to load {filename}: {ex.GetType().Name}: {ex.Message}"); + } else { + log.LogMessage(MessageImportance.Low, $"Skippping {filename}: {ex.Message}"); + } + return new string[0]; + } + } + }; +} diff --git a/src/csharp/Grpc.Tools/GeneratorServices.cs b/src/csharp/Grpc.Tools/GeneratorServices.cs new file mode 100644 index 00000000000..e1f266aa16e --- /dev/null +++ b/src/csharp/Grpc.Tools/GeneratorServices.cs @@ -0,0 +1,168 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools { + // Abstract class for language-specific analysis behavior, such + // as guessing the generated files the same way protoc does. + internal abstract class GeneratorServices { + protected readonly TaskLoggingHelper Log; + protected GeneratorServices(TaskLoggingHelper log) { + Log = log; + } + + // Obtain a service for the given language (csharp, cpp). + public static GeneratorServices GetForLanguage(string lang, TaskLoggingHelper log) { + if (lang.EqualNoCase("csharp")) + return new CSharpGeneratorServices(log); + if (lang.EqualNoCase("cpp")) + return new CppGeneratorServices(log); + log.LogError("Invalid value '{0}' for task property 'Generator'. " + + "Supported generator languages: CSharp, Cpp.", lang); + return null; + } + + // Guess whether item's metadata suggests gRPC stub generation. + // When "gRPCServices" is not defined, assume gRPC is not used. + // When defined, C# uses "none" to skip gRPC, C++ uses "false", so + // recognize both. Since the value is tightly coupled to the scripts, + // we do not try to validate the value; scripts take care of that. + // It is safe to assume that gRPC is requested for any other value. + protected bool GrpcOutputPossible(ITaskItem proto) { + string gsm = proto.GetMetadata(Metadata.kGrpcServices); + return !gsm.EqualNoCase("") && !gsm.EqualNoCase("none") + && !gsm.EqualNoCase("false"); + } + + public abstract string[] GetPossibleOutputs(ITaskItem proto); + }; + + // C# generator services. + internal class CSharpGeneratorServices : GeneratorServices { + public CSharpGeneratorServices(TaskLoggingHelper log) : base(log) {} + + public override string[] GetPossibleOutputs(ITaskItem protoItem) { + bool doGrpc = GrpcOutputPossible(protoItem); + string filename = LowerUnderscoreToUpperCamel( + Path.GetFileNameWithoutExtension(protoItem.ItemSpec)); + + var outputs = new string[doGrpc ? 2 : 1]; + string outdir = protoItem.GetMetadata(Metadata.kOutputDir); + string fileStem = Path.Combine(outdir, filename); + outputs[0] = fileStem + ".cs"; + if (doGrpc) { + // Override outdir if kGrpcOutputDir present, default to proto output. + outdir = protoItem.GetMetadata(Metadata.kGrpcOutputDir); + if (outdir != "") { + fileStem = Path.Combine(outdir, filename); + } + outputs[1] = fileStem + "Grpc.cs"; + } + return outputs; + } + + string LowerUnderscoreToUpperCamel(string str) { + // See src/compiler/generator_helpers.h:118 + var result = new StringBuilder(str.Length, str.Length); + bool cap = true; + foreach (char c in str) { + if (c == '_') { + cap = true; + } else if (cap) { + result.Append(char.ToUpperInvariant(c)); + cap = false; + } else { + result.Append(c); + } + } + return result.ToString(); + } + }; + + // C++ generator services. + internal class CppGeneratorServices : GeneratorServices { + public CppGeneratorServices(TaskLoggingHelper log) : base(log) { } + + public override string[] GetPossibleOutputs(ITaskItem protoItem) { + bool doGrpc = GrpcOutputPossible(protoItem); + string root = protoItem.GetMetadata(Metadata.kProtoRoot); + string proto = protoItem.ItemSpec; + string filename = Path.GetFileNameWithoutExtension(proto); + // E. g., ("foo/", "foo/bar/x.proto") => "bar" + string relative = GetRelativeDir(root, proto); + + var outputs = new string[doGrpc ? 4 : 2]; + string outdir = protoItem.GetMetadata(Metadata.kOutputDir); + string fileStem = Path.Combine(outdir, relative, filename); + outputs[0] = fileStem + ".pb.cc"; + outputs[1] = fileStem + ".pb.h"; + if (doGrpc) { + // Override outdir if kGrpcOutputDir present, default to proto output. + outdir = protoItem.GetMetadata(Metadata.kGrpcOutputDir); + if (outdir != "") { + fileStem = Path.Combine(outdir, relative, filename); + } + outputs[2] = fileStem + "_grpc.pb.cc"; + outputs[3] = fileStem + "_grpc.pb.h"; + } + return outputs; + } + + // Calculate part of proto path relative to root. Protoc is very picky + // about them matching exactly, so can be we. Expect root be exact prefix + // to proto, minus some slash normalization. + string GetRelativeDir(string root, string proto) { + string protoDir = Path.GetDirectoryName(proto); + string rootDir = EndWithSlash(Path.GetDirectoryName(EndWithSlash(root))); + if (rootDir == s_dotSlash) { + // Special case, otherwise we can return "./" instead of "" below! + return protoDir; + } + if (Platform.IsFsCaseInsensitive) { + protoDir = protoDir.ToLowerInvariant(); + rootDir = rootDir.ToLowerInvariant(); + } + protoDir = EndWithSlash(protoDir); + if (!protoDir.StartsWith(rootDir)) { + Log.LogWarning("ProtoBuf item '{0}' has the ProtoRoot metadata '{1}' " + + "which is not prefix to its path. Cannot compute relative path.", + proto, root); + return ""; + } + return protoDir.Substring(rootDir.Length); + } + + // './' or '.\', normalized per system. + static string s_dotSlash = "." + Path.DirectorySeparatorChar; + + static string EndWithSlash(string str) { + if (str == "") { + return s_dotSlash; + } else if (str[str.Length - 1] != '\\' && str[str.Length - 1] != '/') { + return str + Path.DirectorySeparatorChar; + } else { + return str; + } + } + }; +} diff --git a/src/csharp/Grpc.Tools/Grpc.Tools.csproj b/src/csharp/Grpc.Tools/Grpc.Tools.csproj new file mode 100644 index 00000000000..46a6d4670d2 --- /dev/null +++ b/src/csharp/Grpc.Tools/Grpc.Tools.csproj @@ -0,0 +1,95 @@ + + + + + + Protobuf.MSBuild + $(GrpcCsharpVersion) + + netstandard1.3;net40 + + + + + + + ../../../third_party/protobuf/src/google/protobuf/ + + + + ../protoc_plugins/protoc_ + + + ../protoc_plugins/ + + + + <_NetStandard>False + <_NetStandard Condition=" $(TargetFramework.StartsWith('netstandard')) or $(TargetFramework.StartsWith('netcore')) ">True + + + $(DefineConstants);NETSTANDARD + + + + true + ../../../artifacts + + + build\_protobuf\ + true + true + Grpc.Tools + gRPC and Protocol Buffer compiler for managed C# and native C++ projects. + +Add this package to a project that contains .proto files to be compiled to code. +It contains the compilers, include files and project system integration for gRPC +and Protocol buffer service description files necessary to build them on Windows, +Linux and MacOS. Managed runtime is supplied separately in the Grpc.Core package. + Copyright 2018 gRPC authors + gRPC authors + https://github.com/grpc/grpc/blob/master/LICENSE + https://github.com/grpc/grpc + gRPC RPC protocol HTTP/2 + + + + + + + <_ProtoTemp Include="any.proto;api.proto;descriptor.proto;duration.proto;" /> + <_ProtoTemp Include="empty.proto;field_mask.proto;source_context.proto;" /> + <_ProtoTemp Include="struct.proto;timestamp.proto;type.proto;wrappers.proto" /> + <_Asset PackagePath="build/native/include/google/protobuf/" Include="@(_ProtoTemp->'$(Assets_ProtoInclude)%(Identity)')" /> + + + <_Asset PackagePath="build/native/bin/windows/protoc.exe" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> + <_Asset PackagePath="build/native/bin/linux_x86/protoc" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> + <_Asset PackagePath="build/native/bin/linux_x64/protoc" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> + <_Asset PackagePath="build/native/bin/macosx_x86/protoc" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> + <_Asset PackagePath="build/native/bin/macosx_x64/protoc" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> + + + <_Asset PackagePath="build/native/bin/windows/grpc_csharp_plugin.exe" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> + <_Asset PackagePath="build/native/bin/linux_x86/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/linux_x64/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/macosx_x86/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/macosx_x64/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> + + + + + + + + + + + + + + + + + diff --git a/src/csharp/Grpc.Tools/ProtoCompile.cs b/src/csharp/Grpc.Tools/ProtoCompile.cs new file mode 100644 index 00000000000..76c2338ef96 --- /dev/null +++ b/src/csharp/Grpc.Tools/ProtoCompile.cs @@ -0,0 +1,409 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools { + /// + /// Run Google proto compiler (protoc). + /// + /// After a successful run, the task reads the dependency file if specified + /// to be saved by the compiler, and returns its output files. + /// + /// This task (unlike PrepareProtoCompile) does not attempt to guess anything + /// about language-specific behavior of protoc, and therefore can be used for + /// any language outputs. + /// + public class ProtoCompile : ToolTask { +/* + +Usage: /home/kkm/work/protobuf/src/.libs/lt-protoc [OPTION] PROTO_FILES +Parse PROTO_FILES and generate output based on the options given: + -IPATH, --proto_path=PATH Specify the directory in which to search for + imports. May be specified multiple times; + directories will be searched in order. If not + given, the current working directory is used. + --version Show version info and exit. + -h, --help Show this text and exit. + --encode=MESSAGE_TYPE Read a text-format message of the given type + from standard input and write it in binary + to standard output. The message type must + be defined in PROTO_FILES or their imports. + --decode=MESSAGE_TYPE Read a binary message of the given type from + standard input and write it in text format + to standard output. The message type must + be defined in PROTO_FILES or their imports. + --decode_raw Read an arbitrary protocol message from + standard input and write the raw tag/value + pairs in text format to standard output. No + PROTO_FILES should be given when using this + flag. + --descriptor_set_in=FILES Specifies a delimited list of FILES + each containing a FileDescriptorSet (a + protocol buffer defined in descriptor.proto). + The FileDescriptor for each of the PROTO_FILES + provided will be loaded from these + FileDescriptorSets. If a FileDescriptor + appears multiple times, the first occurrence + will be used. + -oFILE, Writes a FileDescriptorSet (a protocol buffer, + --descriptor_set_out=FILE defined in descriptor.proto) containing all of + the input files to FILE. + --include_imports When using --descriptor_set_out, also include + all dependencies of the input files in the + set, so that the set is self-contained. + --include_source_info When using --descriptor_set_out, do not strip + SourceCodeInfo from the FileDescriptorProto. + This results in vastly larger descriptors that + include information about the original + location of each decl in the source file as + well as surrounding comments. + --dependency_out=FILE Write a dependency output file in the format + expected by make. This writes the transitive + set of input file paths to FILE + --error_format=FORMAT Set the format in which to print errors. + FORMAT may be 'gcc' (the default) or 'msvs' + (Microsoft Visual Studio format). + --print_free_field_numbers Print the free field numbers of the messages + defined in the given proto files. Groups share + the same field number space with the parent + message. Extension ranges are counted as + occupied fields numbers. + + --plugin=EXECUTABLE Specifies a plugin executable to use. + Normally, protoc searches the PATH for + plugins, but you may specify additional + executables not in the path using this flag. + Additionally, EXECUTABLE may be of the form + NAME=PATH, in which case the given plugin name + is mapped to the given executable even if + the executable's own name differs. + --cpp_out=OUT_DIR Generate C++ header and source. + --csharp_out=OUT_DIR Generate C# source file. + --java_out=OUT_DIR Generate Java source file. + --javanano_out=OUT_DIR Generate Java Nano source file. + --js_out=OUT_DIR Generate JavaScript source. + --objc_out=OUT_DIR Generate Objective C header and source. + --php_out=OUT_DIR Generate PHP source file. + --python_out=OUT_DIR Generate Python source file. + --ruby_out=OUT_DIR Generate Ruby source file. + @ Read options and filenames from file. If a + relative file path is specified, the file + will be searched in the working directory. + The --proto_path option will not affect how + this argument file is searched. Content of + the file will be expanded in the position of + @ as in the argument list. Note + that shell expansion is not applied to the + content of the file (i.e., you cannot use + quotes, wildcards, escapes, commands, etc.). + Each line corresponds to a single argument, + even if it contains spaces. +*/ + static string[] s_supportedGenerators = new[] { + "cpp", "csharp", "java", + "javanano", "js", "objc", + "php", "python", "ruby", + }; + + /// + /// Code generator. + /// + [Required] + public string Generator { get; set; } + + /// + /// Protobuf files to compile. + /// + [Required] + public ITaskItem[] ProtoBuf { get; set; } + + /// + /// Directory where protoc dependency files are cached. If provided, dependency + /// output filename is autogenerated from source directory hash and file name. + /// Mutually exclusive with DependencyOut. + /// Switch: --dependency_out (with autogenerated file name). + /// + public string ProtoDepDir { get; set; } + + /// + /// Dependency file full name. Mutually exclusive with ProtoDepDir. + /// Autogenerated file name is available in this property after execution. + /// Switch: --dependency_out. + /// + [Output] + public string DependencyOut { get; set; } + + /// + /// The directories to search for imports. Directories will be searched + /// in order. If not given, the current working directory is used. + /// Switch: --proto_path. + /// + public string[] ProtoPath { get; set; } + + /// + /// Generated code directory. The generator property determines the language. + /// Switch: --GEN-out= (for different generators GEN). + /// + [Required] + public string OutputDir { get; set; } + + /// + /// Codegen options. See also OptionsFromMetadata. + /// Switch: --GEN_out= (for different generators GEN). + /// + public string[] OutputOptions { get; set; } + + /// + /// Full path to the gRPC plugin executable. If specified, gRPC generation + /// is enabled for the files. + /// Switch: --plugin=protoc-gen-grpc= + /// + public string GrpcPluginExe { get; set; } + + /// + /// Generated gRPC directory. The generator property determines the + /// language. If gRPC is enabled but this is not given, OutputDir is used. + /// Switch: --grpc_out= + /// + public string GrpcOutputDir { get; set; } + + /// + /// gRPC Codegen options. See also OptionsFromMetadata. + /// --grpc_opt=opt1,opt2=val (comma-separated). + /// + public string[] GrpcOutputOptions { get; set; } + + /// + /// List of files written in addition to generated outputs. Includes a + /// single item for the dependency file if written. + /// + [Output] + public ITaskItem[] AdditionalFileWrites { get; private set; } + + /// + /// List of language files generated by protoc. Empty unless DependencyOut + /// or ProtoDepDir is set, since the file writes are extracted from protoc + /// dependency output file. + /// + [Output] + public ITaskItem[] GeneratedFiles { get; private set; } + + // Hide this property from MSBuild, we should never use a shell script. + private new bool UseCommandProcessor { get; set; } + + protected override string ToolName => + Platform.IsWindows ? "protoc.exe" : "protoc"; + + // Since we never try to really locate protoc.exe somehow, just try ToolExe + // as the full tool location. It will be either just protoc[.exe] from + // ToolName above if not set by the user, or a user-supplied full path. The + // base class will then resolve the former using system PATH. + protected override string GenerateFullPathToTool() => ToolExe; + + // Log protoc errors with the High priority (bold white in MsBuild, + // printed with -v:n, and shown in the Output windows in VS). + protected override MessageImportance StandardErrorLoggingImportance => + MessageImportance.High; + + // Called by base class to validate arguments and make them consistent. + protected override bool ValidateParameters() { + // Part of proto command line switches, must be lowercased. + Generator = Generator.ToLowerInvariant(); + if (!System.Array.Exists(s_supportedGenerators, g => g == Generator)) + Log.LogError("Invalid value for Generator='{0}'. Supported generators: {1}", + Generator, string.Join(", ", s_supportedGenerators)); + + if (ProtoDepDir != null && DependencyOut != null) + Log.LogError("Properties ProtoDepDir and DependencyOut may not be both specified"); + + if (ProtoBuf.Length > 1 && (ProtoDepDir != null || DependencyOut != null)) + Log.LogError("Proto compiler currently allows only one input when " + + "--dependency_out is specified (via ProtoDepDir or DependencyOut). " + + "Tracking issue: https://github.com/google/protobuf/pull/3959"); + + // Use ProtoDepDir to autogenerate DependencyOut + if (ProtoDepDir != null) { + DependencyOut = DepFileUtil.GetDepFilenameForProto(ProtoDepDir, ProtoBuf[0].ItemSpec); + } + + if (GrpcPluginExe == null) { + GrpcOutputOptions = null; + GrpcOutputDir = null; + } else if (GrpcOutputDir == null) { + // Use OutputDir for gRPC output if not specified otherwise by user. + GrpcOutputDir = OutputDir; + } + + return !Log.HasLoggedErrors && base.ValidateParameters(); + } + + // Protoc chokes on BOM, naturally. I would! + static readonly Encoding s_utf8WithoutBom = new UTF8Encoding(false); + protected override Encoding ResponseFileEncoding => s_utf8WithoutBom; + + // Protoc takes one argument per line from the response file, and does not + // require any quoting whatsoever. Otherwise, this is similar to the + // standard CommandLineBuilder + class ProtocResponseFileBuilder { + StringBuilder _data = new StringBuilder(1000); + public override string ToString() => _data.ToString(); + + // If 'value' is not empty, append '--name=value\n'. + public void AddSwitchMaybe(string name, string value) { + if (!string.IsNullOrEmpty(value)) { + _data.Append("--").Append(name).Append("=") + .Append(value).Append('\n'); + } + } + + // Add switch with the 'values' separated by commas, for options. + public void AddSwitchMaybe(string name, string[] values) { + if (values?.Length > 0) { + _data.Append("--").Append(name).Append("=") + .Append(string.Join(",", values)).Append('\n'); + } + } + + // Add a positional argument to the file data. + public void AddArg(string arg) { + _data.Append(arg).Append('\n'); + } + }; + + // Called by the base ToolTask to get response file contents. + protected override string GenerateResponseFileCommands() { + var cmd = new ProtocResponseFileBuilder(); + cmd.AddSwitchMaybe(Generator + "_out", TrimEndSlash(OutputDir)); + cmd.AddSwitchMaybe(Generator + "_opt", OutputOptions); + cmd.AddSwitchMaybe("plugin=protoc-gen-grpc", GrpcPluginExe); + cmd.AddSwitchMaybe("grpc_out", TrimEndSlash(GrpcOutputDir)); + cmd.AddSwitchMaybe("grpc_opt", GrpcOutputOptions); + if (ProtoPath != null) { + foreach (string path in ProtoPath) + cmd.AddSwitchMaybe("proto_path", TrimEndSlash(path)); + } + cmd.AddSwitchMaybe("dependency_out", DependencyOut); + foreach (var proto in ProtoBuf) { + cmd.AddArg(proto.ItemSpec); + } + return cmd.ToString(); + } + + // Protoc cannot digest trailing slashes in directory names, + // curiously under Linux, but not in Windows. + static string TrimEndSlash(string dir) { + if (dir == null || dir.Length <= 1) { + return dir; + } + string trim = dir.TrimEnd('/', '\\'); + // Do not trim the root slash, drive letter possible. + if (trim.Length == 0) { + // Slashes all the way down. + return dir.Substring(0, 1); + } + if (trim.Length == 2 && dir.Length > 2 && trim[1] == ':') { + // We have a drive letter and root, e. g. 'C:\' + return dir.Substring(0, 3); + } + return trim; + } + + // Called by the base class to log tool's command line. + // + // Protoc command file is peculiar, with one argument per line, separated + // by newlines. Unwrap it for log readability into a single line, and also + // quote arguments, lest it look weird and so it may be copied and pasted + // into shell. Since this is for logging only, correct enough is correct. + protected override void LogToolCommand(string cmd) { + var printer = new StringBuilder(1024); + + // Print 'str' slice into 'printer', wrapping in quotes if contains some + // interesting characters in file names, or if empty string. The list of + // characters requiring quoting is not by any means exhaustive; we are + // just striving to be nice, not guaranteeing to be nice. + var quotable = new[] { ' ', '!', '$', '&', '\'', '^' }; + void PrintQuoting(string str, int start, int count) { + bool wrap = count == 0 || str.IndexOfAny(quotable, start, count) >= 0; + if (wrap) printer.Append('"'); + printer.Append(str, start, count); + if (wrap) printer.Append('"'); + } + + for (int ib = 0, ie; (ie = cmd.IndexOf('\n', ib)) >= 0; ib = ie + 1) { + // First line only contains both the program name and the first switch. + // We can rely on at least the '--out_dir' switch being always present. + if (ib == 0) { + int iep = cmd.IndexOf(" --"); + if (iep > 0) { + PrintQuoting(cmd, 0, iep); + ib = iep + 1; + } + } + printer.Append(' '); + if (cmd[ib] == '-') { + // Print switch unquoted, including '=' if any. + int iarg = cmd.IndexOf('=', ib, ie - ib); + if (iarg < 0) { + // Bare switch without a '='. + printer.Append(cmd, ib, ie - ib); + continue; + } + printer.Append(cmd, ib, iarg + 1 - ib); + ib = iarg + 1; + } + // A positional argument or switch value. + PrintQuoting(cmd, ib, ie - ib); + } + + base.LogToolCommand(printer.ToString()); + } + + // Main task entry point. + public override bool Execute() { + base.UseCommandProcessor = false; + + bool ok = base.Execute(); + if (!ok) { + return false; + } + + // Read dependency output file from the compiler to retrieve the + // definitive list of created files. Report the dependency file + // itself as having been written to. + if (DependencyOut != null) { + string[] outputs = DepFileUtil.ReadDependencyOutputs(DependencyOut, Log); + if (HasLoggedErrors) { + return false; + } + + GeneratedFiles = new ITaskItem[outputs.Length]; + for (int i = 0; i < outputs.Length; i++) { + GeneratedFiles[i] = new TaskItem(outputs[i]); + } + AdditionalFileWrites = new ITaskItem[] { + new TaskItem(DependencyOut) + }; + } + + return true; + } + }; +} diff --git a/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs b/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs new file mode 100644 index 00000000000..9afea9255e5 --- /dev/null +++ b/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs @@ -0,0 +1,80 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Collections.Generic; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools { + public class ProtoCompilerOutputs : Task { + /// + /// Code generator. Currently supported are "csharp", "cpp". + /// + [Required] + public string Generator { get; set; } + + /// + /// All Proto files in the project. The task computes possible outputs + /// from these proto files, and returns them in the PossibleOutputs list. + /// Not all of these might be actually produced by protoc; this is dealt + /// with later in the ProtoCompile task which returns the list of + /// files actually produced by the compiler. + /// + [Required] + public ITaskItem[] ProtoBuf { get; set; } + + /// + /// Output items per each potential output. We do not look at existing + /// cached dependency even if they exist, since file may be refactored, + /// affecting whether or not gRPC code file is generated from a given proto. + /// Instead, all potentially possible generated sources are collected. + /// It is a wise idea to generate empty files later for those potentials + /// that are not actually created by protoc, so the dependency checks + /// result in a minimal recompilation. The Protoc task can output the + /// list of files it actually produces, given right combination of its + /// properties. + /// Output items will have the Source metadata set on them: + /// + /// + [Output] + public ITaskItem[] PossibleOutputs { get; private set; } + + public override bool Execute() { + var generator = GeneratorServices.GetForLanguage(Generator, Log); + if (generator == null) { + // Error already logged, just return. + return false; + } + + // Get language-specific possible output. The generator expects certain + // metadata be set on the proto item. + var possible = new List(); + foreach (var proto in ProtoBuf) { + var outputs = generator.GetPossibleOutputs(proto); + foreach (string output in outputs) { + var ti = new TaskItem(output); + ti.SetMetadata(Metadata.kSource, proto.ItemSpec); + possible.Add(ti); + } + } + PossibleOutputs = possible.ToArray(); + + return !Log.HasLoggedErrors; + } + }; +} diff --git a/src/csharp/Grpc.Tools/ProtoReadDependencies.cs b/src/csharp/Grpc.Tools/ProtoReadDependencies.cs new file mode 100644 index 00000000000..2ee03891469 --- /dev/null +++ b/src/csharp/Grpc.Tools/ProtoReadDependencies.cs @@ -0,0 +1,70 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Collections.Generic; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools { + public class ProtoReadDependencies : Task { + /// + /// The collection is used to collect possible additional dependencies + /// of proto files cached under ProtoDepDir. + /// + [Required] + public ITaskItem[] ProtoBuf { get; set; } + + /// + /// Directory where protoc dependency files are cached. + /// + [Required] + public string ProtoDepDir { get; set; } + + /// + /// Additional items that a proto file depends on. This list may include + /// extra dependencies; we do our best to include as few extra positives + /// as reasonable to avoid missing any. The collection item is the + /// dependency, and its Source metadatum is the dependent proto file, like + /// + /// + [Output] + public ITaskItem[] Dependencies { get; private set; } + + public override bool Execute() { + // Read dependency files, where available. There might be none, + // just use a best effort. + if (ProtoDepDir != null) { + var dependencies = new List(); + foreach (var proto in ProtoBuf) { + string[] deps = DepFileUtil.ReadDependencyInputs(ProtoDepDir, proto.ItemSpec, Log); + foreach (string dep in deps) { + var ti = new TaskItem(dep); + ti.SetMetadata(Metadata.kSource, proto.ItemSpec); + dependencies.Add(ti); + } + } + Dependencies = dependencies.ToArray(); + } else { + Dependencies = new ITaskItem[0]; + } + + return !Log.HasLoggedErrors; + } + }; +} diff --git a/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs b/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs new file mode 100644 index 00000000000..f505b86fe41 --- /dev/null +++ b/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs @@ -0,0 +1,58 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Grpc.Tools { + /// + /// A helper task to resolve actual OS type and bitness. + /// + public class ProtoToolsPlatform : Task { + /// + /// Return one of 'linux', 'macosx' or 'windows'. + /// If the OS is unknown, the property is not set. + /// + [Output] + public string Os { get; set; } + + /// + /// Return one of 'x64' or 'x86'. + /// If the CPU is unknown, the property is not set. + /// + [Output] + public string Cpu { get; set; } + + + public override bool Execute() { + switch (Platform.Os) { + case Platform.OsKind.Linux: Os = "linux"; break; + case Platform.OsKind.MacOsX: Os = "macosx"; break; + case Platform.OsKind.Windows: Os = "windows"; break; + default: Os = ""; break; + } + + switch (Platform.Cpu) { + case Platform.CpuKind.X86: Cpu = "x86"; break; + case Platform.CpuKind.X64: Cpu = "x64"; break; + default: Cpu = ""; break; + } + return true; + } + }; +} diff --git a/src/csharp/Grpc.Tools/build/Grpc.Tools.props b/src/csharp/Grpc.Tools/build/Grpc.Tools.props new file mode 100644 index 00000000000..dbcd8bf494b --- /dev/null +++ b/src/csharp/Grpc.Tools/build/Grpc.Tools.props @@ -0,0 +1,11 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + + + + + diff --git a/src/csharp/Grpc.Tools/build/Grpc.Tools.targets b/src/csharp/Grpc.Tools/build/Grpc.Tools.targets new file mode 100644 index 00000000000..c0a5b1e2c5d --- /dev/null +++ b/src/csharp/Grpc.Tools/build/Grpc.Tools.targets @@ -0,0 +1,11 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + + + + + diff --git a/src/csharp/Grpc.Tools/build/_grpc/Grpc.CSharp.xml b/src/csharp/Grpc.Tools/build/_grpc/Grpc.CSharp.xml new file mode 100644 index 00000000000..54468eb5eff --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_grpc/Grpc.CSharp.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/csharp/Grpc.Tools/build/_grpc/README b/src/csharp/Grpc.Tools/build/_grpc/README new file mode 100644 index 00000000000..4a7204b9ff3 --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_grpc/README @@ -0,0 +1,3 @@ +TODO(kkm): These file will go into Grpc.Tools/build after package split. + Remove leading underscores from file names; they are hiding the + files from some NuGet versions which pull them into project. diff --git a/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.props b/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.props new file mode 100644 index 00000000000..8ce07c48aba --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.props @@ -0,0 +1,6 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + diff --git a/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets b/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets new file mode 100644 index 00000000000..0042bf2bfa9 --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets @@ -0,0 +1,46 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + grpc_csharp_plugin + + + + + + File;BrowseObject + + + + + + Both + + + + + + + $(Protobuf_PackagedToolsPath)bin\$(Protobuf_ToolsOs)\$(gRPC_PluginFileName).exe + $(Protobuf_PackagedToolsPath)bin/$(Protobuf_ToolsOs)_$(Protobuf_ToolsCpu)/$(gRPC_PluginFileName) + + + + + + + $(gRPC_PluginFullPath) + %(Protobuf_Compile.OutputDir) + <_GrpcOutputOptions Condition=" '%(Protobuf_Compile.Access)' == 'Internal' ">%(Protobuf_Compile._GrpcOutputOptions);internal_access + + + <_GrpcOutputOptions>%(Protobuf_Compile._GrpcOutputOptions);no_server + + + <_GrpcOutputOptions>%(Protobuf_Compile._GrpcOutputOptions);no_client + + + + diff --git a/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props new file mode 100644 index 00000000000..06ee9bcda8a --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props @@ -0,0 +1,23 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + 1 + + + $( [System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)../native/) ) + $(Protobuf_PackagedToolsPath)include + + + + + + + diff --git a/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets new file mode 100644 index 00000000000..5a8d3f20276 --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets @@ -0,0 +1,383 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + CSharp + + true + <_Protobuf_MsBuildAssembly Condition=" '$(MSBuildRuntimeType)' == 'Core' ">netstandard1.3\Protobuf.MSBuild.dll + <_Protobuf_MsBuildAssembly Condition=" '$(MSBuildRuntimeType)' != 'Core' ">net40\Protobuf.MSBuild.dll + + + + + + + + + $(IntermediateOutputPath) + $(Protobuf_IntermediatePath) + $(Protobuf_IntermediatePath) + + + + + Public + True + + True + $(Protobuf_OutputPath) + + + + + + File;BrowseObject + + + + + + + false + + + + + + + + + + + + + + + + + + + $(PROTOBUF_TOOLS_OS) + $(PROTOBUF_TOOLS_CPU) + $(PROTOBUF_PROTOC) + + + $(_Protobuf_ToolsOs) + $(_Protobuf_ToolsCpu) + $(Protobuf_PackagedToolsPath)bin\$(Protobuf_ToolsOs)\protoc.exe + $(Protobuf_PackagedToolsPath)bin/$(Protobuf_ToolsOs)_$(Protobuf_ToolsCpu)/protoc + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + . + + + + %(RelativeDir) + + + + + + %(Identity) + + + + + + + + + + + + + + + + + + + + <_Protobuf_CodeCompile Include="@(Protobuf_ExpectedOutputs->Distinct())" + Condition=" '%(Source)' != '' and '@(Protobuf_Compile->WithMetadataValue('CompileOutputs', 'true'))' != '' " /> + + + + + + + + + + %(Identity) + + + + + + + + + + + + + + + + + <_Protobuf_OutOfDateProto Remove="@(_Protobuf_OutOfDateProto->WithMetadataValue('_Exec',''))" /> + + + + + + + <_Protobuf_OutOfDateProto Include="@(Protobuf_Compile)" + Condition = " '%(Source)' != '' and '@(Protobuf_ExpectedOutputs)' == '' "> + <_Exec>true + + + + + + + + + + + + + + + + + + + <_OutputOptions Condition=" '%(Protobuf_Compile.Access)' == 'Internal' ">%(Protobuf_Compile._OutputOptions);internal_access + + + + + + + + + + + + + <_Protobuf_OutOfDateProto> + . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_Protobuf_Protodep Include="$(Protobuf_DepFilesPath)*.protodep" /> + + + + + + + + + + + + + diff --git a/src/csharp/Grpc.Tools/build/_protobuf/Protobuf.CSharp.xml b/src/csharp/Grpc.Tools/build/_protobuf/Protobuf.CSharp.xml new file mode 100644 index 00000000000..2c41fbcbd06 --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_protobuf/Protobuf.CSharp.xml @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/csharp/Grpc.Tools/build/_protobuf/README b/src/csharp/Grpc.Tools/build/_protobuf/README new file mode 100644 index 00000000000..e6e358a218d --- /dev/null +++ b/src/csharp/Grpc.Tools/build/_protobuf/README @@ -0,0 +1 @@ +TODO(kkm): These file will go into Google.Protobuf.Tools/build after package split. diff --git a/src/csharp/Grpc.Tools/build/native/Grpc.Tools.props b/src/csharp/Grpc.Tools/build/native/Grpc.Tools.props new file mode 100644 index 00000000000..7f64ae91658 --- /dev/null +++ b/src/csharp/Grpc.Tools/build/native/Grpc.Tools.props @@ -0,0 +1,15 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + 1 + + + $(MSBuildThisFileDirectory)bin\windows\protoc.exe + $(MSBuildThisFileDirectory)bin\include\ + grpc_cpp_plugin + $(MSBuildThisFileDirectory)bin\windows\grpc_cpp_plugin.exe + + diff --git a/src/csharp/Grpc.sln b/src/csharp/Grpc.sln index d9a7b8d556b..6c1b2e99980 100644 --- a/src/csharp/Grpc.sln +++ b/src/csharp/Grpc.sln @@ -39,6 +39,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Grpc.Reflection.Tests", "Gr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Grpc.Microbenchmarks", "Grpc.Microbenchmarks\Grpc.Microbenchmarks.csproj", "{84C17746-4727-4290-8E8B-A380793DAE1E}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Grpc.Tools", "Grpc.Tools\Grpc.Tools.csproj", "{8A643A1B-B85C-4E3D-BFD3-719FE04D7E91}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Grpc.Tools.Tests", "Grpc.Tools.Tests\Grpc.Tools.Tests.csproj", "{AEBE9BD8-E433-45B7-8B3D-D458EDBBCFC4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -117,6 +121,14 @@ Global {84C17746-4727-4290-8E8B-A380793DAE1E}.Debug|Any CPU.Build.0 = Debug|Any CPU {84C17746-4727-4290-8E8B-A380793DAE1E}.Release|Any CPU.ActiveCfg = Release|Any CPU {84C17746-4727-4290-8E8B-A380793DAE1E}.Release|Any CPU.Build.0 = Release|Any CPU + {8A643A1B-B85C-4E3D-BFD3-719FE04D7E91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A643A1B-B85C-4E3D-BFD3-719FE04D7E91}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A643A1B-B85C-4E3D-BFD3-719FE04D7E91}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A643A1B-B85C-4E3D-BFD3-719FE04D7E91}.Release|Any CPU.Build.0 = Release|Any CPU + {AEBE9BD8-E433-45B7-8B3D-D458EDBBCFC4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AEBE9BD8-E433-45B7-8B3D-D458EDBBCFC4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AEBE9BD8-E433-45B7-8B3D-D458EDBBCFC4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AEBE9BD8-E433-45B7-8B3D-D458EDBBCFC4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 394b859e0be..cae253bc7b9 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -45,10 +45,10 @@ xcopy /Y /I nativelibs\csharp_ext_windows_x64\grpc_csharp_ext.dll ..\..\cmake\bu %DOTNET% pack --configuration Release Grpc.Auth --output ..\..\..\artifacts || goto :error %DOTNET% pack --configuration Release Grpc.HealthCheck --output ..\..\..\artifacts || goto :error %DOTNET% pack --configuration Release Grpc.Reflection --output ..\..\..\artifacts || goto :error +%DOTNET% pack --configuration Release Grpc.Tools --output ..\..\..\artifacts || goto :error %NUGET% pack Grpc.nuspec -Version %VERSION% -OutputDirectory ..\..\artifacts || goto :error %NUGET% pack Grpc.Core.NativeDebug.nuspec -Version %VERSION% -OutputDirectory ..\..\artifacts -%NUGET% pack Grpc.Tools.nuspec -Version %VERSION% -OutputDirectory ..\..\artifacts @rem copy resulting nuget packages to artifacts directory xcopy /Y /I *.nupkg ..\..\artifacts\ || goto :error diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 273d745f170..8b81eba3aaa 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -44,9 +44,9 @@ dotnet pack --configuration Release Grpc.Core.Testing --output ../../../artifact dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts +dotnet pack --configuration Release Grpc.Tools --output ../../../artifacts nuget pack Grpc.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/csharp/tests.json b/src/csharp/tests.json index c2f243fe0ae..483d1d7aada 100644 --- a/src/csharp/tests.json +++ b/src/csharp/tests.json @@ -62,5 +62,15 @@ "Grpc.Reflection.Tests": [ "Grpc.Reflection.Tests.ReflectionClientServerTest", "Grpc.Reflection.Tests.SymbolRegistryTest" + ], + "Grpc.Tools.Tests": [ + "Grpc.Tools.Tests.CppGeneratorTests", + "Grpc.Tools.Tests.CSharpGeneratorTests", + "Grpc.Tools.Tests.GeneratorTests", + "Grpc.Tools.Tests.ProtoCompileBasicTests", + "Grpc.Tools.Tests.ProtoCompileCommandLineGeneratorTests", + "Grpc.Tools.Tests.ProtoCompileCommandLinePrinterTests", + "Grpc.Tools.Tests.ProtoToolsPlatformTaskTests", + "Grps.Tools.Tests.DepFileUtilTests" ] } diff --git a/templates/src/csharp/build_packages_dotnetcli.bat.template b/templates/src/csharp/build_packages_dotnetcli.bat.template index 45010fec741..cdadbba44f2 100755 --- a/templates/src/csharp/build_packages_dotnetcli.bat.template +++ b/templates/src/csharp/build_packages_dotnetcli.bat.template @@ -47,10 +47,10 @@ %%DOTNET% pack --configuration Release Grpc.Auth --output ..\..\..\artifacts || goto :error %%DOTNET% pack --configuration Release Grpc.HealthCheck --output ..\..\..\artifacts || goto :error %%DOTNET% pack --configuration Release Grpc.Reflection --output ..\..\..\artifacts || goto :error + %%DOTNET% pack --configuration Release Grpc.Tools --output ..\..\..\artifacts || goto :error %%NUGET% pack Grpc.nuspec -Version %VERSION% -OutputDirectory ..\..\artifacts || goto :error %%NUGET% pack Grpc.Core.NativeDebug.nuspec -Version %VERSION% -OutputDirectory ..\..\artifacts - %%NUGET% pack Grpc.Tools.nuspec -Version %VERSION% -OutputDirectory ..\..\artifacts @rem copy resulting nuget packages to artifacts directory xcopy /Y /I *.nupkg ..\..\artifacts\ || goto :error diff --git a/templates/src/csharp/build_packages_dotnetcli.sh.template b/templates/src/csharp/build_packages_dotnetcli.sh.template index 1172582ebde..5eba62efabf 100755 --- a/templates/src/csharp/build_packages_dotnetcli.sh.template +++ b/templates/src/csharp/build_packages_dotnetcli.sh.template @@ -46,9 +46,9 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts + dotnet pack --configuration Release Grpc.Tools --output ../../../artifacts nuget pack Grpc.nuspec -Version "${settings.csharp_version}" -OutputDirectory ../../artifacts nuget pack Grpc.Core.NativeDebug.nuspec -Version "${settings.csharp_version}" -OutputDirectory ../../artifacts - nuget pack Grpc.Tools.nuspec -Version "${settings.csharp_version}" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) From 17df1f8cf59b88ca6df78ebc736ae038dd3bbfbe Mon Sep 17 00:00:00 2001 From: kkm Date: Wed, 13 Jun 2018 18:27:07 -0700 Subject: [PATCH 02/53] fixup! Add Grpc.Tools MsBuild taks assembly, test and scripting --- .../Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 14 +- src/csharp/Grpc.Tools/Common.cs | 12 +- src/csharp/Grpc.Tools/DepFileUtil.cs | 103 +++++++---- src/csharp/Grpc.Tools/GeneratorServices.cs | 12 +- src/csharp/Grpc.Tools/Grpc.Tools.csproj | 31 ++-- src/csharp/Grpc.Tools/ProtoCompile.cs | 170 +++++++++--------- src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs | 2 +- .../Grpc.Tools/ProtoReadDependencies.cs | 2 +- .../_protobuf/Google.Protobuf.Tools.targets | 4 +- 9 files changed, 201 insertions(+), 149 deletions(-) diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index 585e4518b59..661d97d81bc 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -1,23 +1,25 @@ + + - netcoreapp1.0;net45 + net45;netcoreapp1.0 Exe + + - - - + + - - + diff --git a/src/csharp/Grpc.Tools/Common.cs b/src/csharp/Grpc.Tools/Common.cs index df539f8c4f9..1ebd386bd1c 100644 --- a/src/csharp/Grpc.Tools/Common.cs +++ b/src/csharp/Grpc.Tools/Common.cs @@ -25,15 +25,15 @@ using System.Security; [assembly: InternalsVisibleTo("Grpc.Tools.Tests")] namespace Grpc.Tools { - // Metadata names that we refer to often. + // Metadata names (MSBuild item attributes) that we refer to often. static class Metadata { // On output dependency lists. - public static string kSource = "Source"; + public static string Source = "Source"; // On ProtoBuf items. - public static string kProtoRoot = "ProtoRoot"; - public static string kOutputDir = "OutputDir"; - public static string kGrpcServices = "GrpcServices"; - public static string kGrpcOutputDir = "GrpcOutputDir"; + public static string ProtoRoot = "ProtoRoot"; + public static string OutputDir = "OutputDir"; + public static string GrpcServices = "GrpcServices"; + public static string GrpcOutputDir = "GrpcOutputDir"; }; // A few flags used to control the behavior under various platforms. diff --git a/src/csharp/Grpc.Tools/DepFileUtil.cs b/src/csharp/Grpc.Tools/DepFileUtil.cs index 2a931b7295d..e635ad1e852 100644 --- a/src/csharp/Grpc.Tools/DepFileUtil.cs +++ b/src/csharp/Grpc.Tools/DepFileUtil.cs @@ -25,29 +25,38 @@ using Microsoft.Build.Utilities; namespace Grpc.Tools { internal static class DepFileUtil { -/* - Sample dependency files. Notable features we have to deal with: - * Slash doubling, must normalize them. - * Spaces in file names. Cannot just "unwrap" the line on backslash at eof; - rather, treat every line as containing one file name except for one with - the ':' separator, as containing exactly two. - * Deal with ':' also being drive letter separator (second example). - -obj\Release\net45\/Foo.cs \ -obj\Release\net45\/FooGrpc.cs: C:/foo/include/google/protobuf/wrappers.proto\ - C:/projects/foo/src//foo.proto - -C:\projects\foo\src\./foo.grpc.pb.cc \ -C:\projects\foo\src\./foo.grpc.pb.h \ -C:\projects\foo\src\./foo.pb.cc \ -C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ - C:/foo/include/google/protobuf/any.proto\ - C:/foo/include/google/protobuf/source_context.proto\ - C:/foo/include/google/protobuf/type.proto\ - foo.proto -*/ - - // Read file names from the dependency file to the right of ':'. + /* + Sample dependency files. Notable features we have to deal with: + * Slash doubling, must normalize them. + * Spaces in file names. Cannot just "unwrap" the line on backslash at eof; + rather, treat every line as containing one file name except for one with + the ':' separator, as containing exactly two. + * Deal with ':' also being drive letter separator (second example). + + obj\Release\net45\/Foo.cs \ + obj\Release\net45\/FooGrpc.cs: C:/foo/include/google/protobuf/wrappers.proto\ + C:/projects/foo/src//foo.proto + + C:\projects\foo\src\./foo.grpc.pb.cc \ + C:\projects\foo\src\./foo.grpc.pb.h \ + C:\projects\foo\src\./foo.pb.cc \ + C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ + C:/foo/include/google/protobuf/any.proto\ + C:/foo/include/google/protobuf/source_context.proto\ + C:/foo/include/google/protobuf/type.proto\ + foo.proto + */ + + /// + /// Read file names from the dependency file to the right of ':' + /// + /// Relative path to the dependency cache, e. g. "out" + /// Relative path to the proto item, e. g. "foo/file.proto" + /// A for logging + /// + /// Array of the proto file input dependencies as written by protoc, or empty + /// array if the dependency file does not exist or cannot be parsed. + /// public static string[] ReadDependencyInputs(string protoDepDir, string proto, TaskLoggingHelper log) { string depFilename = GetDepFilenameForProto(protoDepDir, proto); @@ -80,7 +89,20 @@ C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ return result.ToArray(); } - // Read file names from the dependency file to the left of ':'. + /// + /// Read file names from the dependency file to the left of ':' + /// + /// Path to dependency file written by protoc + /// A for logging + /// + /// Array of the protoc-generated outputs from the given dependency file + /// written by protoc, or empty array if the file does not exist or cannot + /// be parsed. + /// + /// + /// Since this is called after a protoc invocation, an unparsable or missing + /// file causes an error-level message to be logged. + /// public static string[] ReadDependencyOutputs(string depFilename, TaskLoggingHelper log) { string[] lines = ReadDepFileLines(depFilename, true, log); @@ -106,10 +128,31 @@ C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ return result.ToArray(); } - // Get complete dependency file name from directory hash and file name, - // tucked onto protoDepDir, e. g. - // ("out", "foo/file.proto") => "out/deadbeef12345678_file.protodep". - // This way, the filenames are unique but still possible to make sense of. + /// + /// Construct relative dependency file name from directory hash and file name + /// + /// Relative path to the dependency cache, e. g. "out" + /// Relative path to the proto item, e. g. "foo/file.proto" + /// + /// Full relative path to the dependency file, e. g. + /// "out/deadbeef12345678_file.protodep" + /// + /// + /// Since a project may contain proto files with the same filename but in different + /// directories, a unique filename for the dependency file is constructed based on the + /// proto file name both name and directory. The directory path can be arbitrary, + /// for example, it can be outside of the project, or an absolute path including + /// a drive letter, or a UNC network path. A name constructed from such a path by, + /// for example, replacing disallowed name characters with an underscore, may well + /// be over filesystem's allowed path length, since it will be located under the + /// project and solution directories, which are also some level deep from the root. + /// Instead of creating long and unwieldy names for these proto sources, we cache + /// the full path of the name without the filename, and append the filename to it, + /// as in e. g. "foo/file.proto" will yield the name "deadbeef12345678_file", where + /// "deadbeef12345678" is a presumed hash value of the string "foo/". This allows + /// the file names be short, unique (up to a hash collision), and still allowing + /// the user to guess their provenance. + /// public static string GetDepFilenameForProto(string protoDepDir, string proto) { string dirname = Path.GetDirectoryName(proto); if (Platform.IsFsCaseInsensitive) { @@ -177,7 +220,7 @@ C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ // Read entire dependency file. The 'required' parameter controls error // logging behavior in case the file not found. We require this file when - // compiling, but reading it is optional when computing depnedencies. + // compiling, but reading it is optional when computing dependencies. static string[] ReadDepFileLines(string filename, bool required, TaskLoggingHelper log) { try { @@ -189,7 +232,7 @@ C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ if (required) { log.LogError($"Unable to load {filename}: {ex.GetType().Name}: {ex.Message}"); } else { - log.LogMessage(MessageImportance.Low, $"Skippping {filename}: {ex.Message}"); + log.LogMessage(MessageImportance.Low, $"Skipping {filename}: {ex.Message}"); } return new string[0]; } diff --git a/src/csharp/Grpc.Tools/GeneratorServices.cs b/src/csharp/Grpc.Tools/GeneratorServices.cs index e1f266aa16e..52bd29a6781 100644 --- a/src/csharp/Grpc.Tools/GeneratorServices.cs +++ b/src/csharp/Grpc.Tools/GeneratorServices.cs @@ -49,7 +49,7 @@ namespace Grpc.Tools { // we do not try to validate the value; scripts take care of that. // It is safe to assume that gRPC is requested for any other value. protected bool GrpcOutputPossible(ITaskItem proto) { - string gsm = proto.GetMetadata(Metadata.kGrpcServices); + string gsm = proto.GetMetadata(Metadata.GrpcServices); return !gsm.EqualNoCase("") && !gsm.EqualNoCase("none") && !gsm.EqualNoCase("false"); } @@ -67,12 +67,12 @@ namespace Grpc.Tools { Path.GetFileNameWithoutExtension(protoItem.ItemSpec)); var outputs = new string[doGrpc ? 2 : 1]; - string outdir = protoItem.GetMetadata(Metadata.kOutputDir); + string outdir = protoItem.GetMetadata(Metadata.OutputDir); string fileStem = Path.Combine(outdir, filename); outputs[0] = fileStem + ".cs"; if (doGrpc) { // Override outdir if kGrpcOutputDir present, default to proto output. - outdir = protoItem.GetMetadata(Metadata.kGrpcOutputDir); + outdir = protoItem.GetMetadata(Metadata.GrpcOutputDir); if (outdir != "") { fileStem = Path.Combine(outdir, filename); } @@ -105,20 +105,20 @@ namespace Grpc.Tools { public override string[] GetPossibleOutputs(ITaskItem protoItem) { bool doGrpc = GrpcOutputPossible(protoItem); - string root = protoItem.GetMetadata(Metadata.kProtoRoot); + string root = protoItem.GetMetadata(Metadata.ProtoRoot); string proto = protoItem.ItemSpec; string filename = Path.GetFileNameWithoutExtension(proto); // E. g., ("foo/", "foo/bar/x.proto") => "bar" string relative = GetRelativeDir(root, proto); var outputs = new string[doGrpc ? 4 : 2]; - string outdir = protoItem.GetMetadata(Metadata.kOutputDir); + string outdir = protoItem.GetMetadata(Metadata.OutputDir); string fileStem = Path.Combine(outdir, relative, filename); outputs[0] = fileStem + ".pb.cc"; outputs[1] = fileStem + ".pb.h"; if (doGrpc) { // Override outdir if kGrpcOutputDir present, default to proto output. - outdir = protoItem.GetMetadata(Metadata.kGrpcOutputDir); + outdir = protoItem.GetMetadata(Metadata.GrpcOutputDir); if (outdir != "") { fileStem = Path.Combine(outdir, relative, filename); } diff --git a/src/csharp/Grpc.Tools/Grpc.Tools.csproj b/src/csharp/Grpc.Tools/Grpc.Tools.csproj index 46a6d4670d2..8edfb848d72 100644 --- a/src/csharp/Grpc.Tools/Grpc.Tools.csproj +++ b/src/csharp/Grpc.Tools/Grpc.Tools.csproj @@ -6,7 +6,19 @@ Protobuf.MSBuild $(GrpcCsharpVersion) - netstandard1.3;net40 + net45;netstandard1.3 + + + + + + /usr/lib/mono/4.5-api + /usr/local/lib/mono/4.5-api + /Library/Frameworks/Mono.framework/Versions/Current/lib/mono/4.5-api @@ -32,9 +44,6 @@ - true - ../../../artifacts - build\_protobuf\ true @@ -57,10 +66,9 @@ Linux and MacOS. Managed runtime is supplied separately in the Grpc.Core package - <_ProtoTemp Include="any.proto;api.proto;descriptor.proto;duration.proto;" /> - <_ProtoTemp Include="empty.proto;field_mask.proto;source_context.proto;" /> - <_ProtoTemp Include="struct.proto;timestamp.proto;type.proto;wrappers.proto" /> - <_Asset PackagePath="build/native/include/google/protobuf/" Include="@(_ProtoTemp->'$(Assets_ProtoInclude)%(Identity)')" /> + <_ProtoAssetName Include="any;api;descriptor;duration;empty;field_mask; + source_context;struct;timestamp;type;wrappers" /> + <_Asset PackagePath="build/native/include/google/protobuf/" Include="@(_ProtoAssetName->'$(Assets_ProtoInclude)%(Identity).proto')" /> <_Asset PackagePath="build/native/bin/windows/protoc.exe" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> @@ -85,10 +93,9 @@ Linux and MacOS. Managed runtime is supplied separately in the Grpc.Core package - - - + + diff --git a/src/csharp/Grpc.Tools/ProtoCompile.cs b/src/csharp/Grpc.Tools/ProtoCompile.cs index 76c2338ef96..e77084b1ef0 100644 --- a/src/csharp/Grpc.Tools/ProtoCompile.cs +++ b/src/csharp/Grpc.Tools/ProtoCompile.cs @@ -32,91 +32,91 @@ namespace Grpc.Tools { /// any language outputs. /// public class ProtoCompile : ToolTask { -/* - -Usage: /home/kkm/work/protobuf/src/.libs/lt-protoc [OPTION] PROTO_FILES -Parse PROTO_FILES and generate output based on the options given: - -IPATH, --proto_path=PATH Specify the directory in which to search for - imports. May be specified multiple times; - directories will be searched in order. If not - given, the current working directory is used. - --version Show version info and exit. - -h, --help Show this text and exit. - --encode=MESSAGE_TYPE Read a text-format message of the given type - from standard input and write it in binary - to standard output. The message type must - be defined in PROTO_FILES or their imports. - --decode=MESSAGE_TYPE Read a binary message of the given type from - standard input and write it in text format - to standard output. The message type must - be defined in PROTO_FILES or their imports. - --decode_raw Read an arbitrary protocol message from - standard input and write the raw tag/value - pairs in text format to standard output. No - PROTO_FILES should be given when using this - flag. - --descriptor_set_in=FILES Specifies a delimited list of FILES - each containing a FileDescriptorSet (a - protocol buffer defined in descriptor.proto). - The FileDescriptor for each of the PROTO_FILES - provided will be loaded from these - FileDescriptorSets. If a FileDescriptor - appears multiple times, the first occurrence - will be used. - -oFILE, Writes a FileDescriptorSet (a protocol buffer, - --descriptor_set_out=FILE defined in descriptor.proto) containing all of - the input files to FILE. - --include_imports When using --descriptor_set_out, also include - all dependencies of the input files in the - set, so that the set is self-contained. - --include_source_info When using --descriptor_set_out, do not strip - SourceCodeInfo from the FileDescriptorProto. - This results in vastly larger descriptors that - include information about the original - location of each decl in the source file as - well as surrounding comments. - --dependency_out=FILE Write a dependency output file in the format - expected by make. This writes the transitive - set of input file paths to FILE - --error_format=FORMAT Set the format in which to print errors. - FORMAT may be 'gcc' (the default) or 'msvs' - (Microsoft Visual Studio format). - --print_free_field_numbers Print the free field numbers of the messages - defined in the given proto files. Groups share - the same field number space with the parent - message. Extension ranges are counted as - occupied fields numbers. - - --plugin=EXECUTABLE Specifies a plugin executable to use. - Normally, protoc searches the PATH for - plugins, but you may specify additional - executables not in the path using this flag. - Additionally, EXECUTABLE may be of the form - NAME=PATH, in which case the given plugin name - is mapped to the given executable even if - the executable's own name differs. - --cpp_out=OUT_DIR Generate C++ header and source. - --csharp_out=OUT_DIR Generate C# source file. - --java_out=OUT_DIR Generate Java source file. - --javanano_out=OUT_DIR Generate Java Nano source file. - --js_out=OUT_DIR Generate JavaScript source. - --objc_out=OUT_DIR Generate Objective C header and source. - --php_out=OUT_DIR Generate PHP source file. - --python_out=OUT_DIR Generate Python source file. - --ruby_out=OUT_DIR Generate Ruby source file. - @ Read options and filenames from file. If a - relative file path is specified, the file - will be searched in the working directory. - The --proto_path option will not affect how - this argument file is searched. Content of - the file will be expanded in the position of - @ as in the argument list. Note - that shell expansion is not applied to the - content of the file (i.e., you cannot use - quotes, wildcards, escapes, commands, etc.). - Each line corresponds to a single argument, - even if it contains spaces. -*/ + /* + + Usage: /home/kkm/work/protobuf/src/.libs/lt-protoc [OPTION] PROTO_FILES + Parse PROTO_FILES and generate output based on the options given: + -IPATH, --proto_path=PATH Specify the directory in which to search for + imports. May be specified multiple times; + directories will be searched in order. If not + given, the current working directory is used. + --version Show version info and exit. + -h, --help Show this text and exit. + --encode=MESSAGE_TYPE Read a text-format message of the given type + from standard input and write it in binary + to standard output. The message type must + be defined in PROTO_FILES or their imports. + --decode=MESSAGE_TYPE Read a binary message of the given type from + standard input and write it in text format + to standard output. The message type must + be defined in PROTO_FILES or their imports. + --decode_raw Read an arbitrary protocol message from + standard input and write the raw tag/value + pairs in text format to standard output. No + PROTO_FILES should be given when using this + flag. + --descriptor_set_in=FILES Specifies a delimited list of FILES + each containing a FileDescriptorSet (a + protocol buffer defined in descriptor.proto). + The FileDescriptor for each of the PROTO_FILES + provided will be loaded from these + FileDescriptorSets. If a FileDescriptor + appears multiple times, the first occurrence + will be used. + -oFILE, Writes a FileDescriptorSet (a protocol buffer, + --descriptor_set_out=FILE defined in descriptor.proto) containing all of + the input files to FILE. + --include_imports When using --descriptor_set_out, also include + all dependencies of the input files in the + set, so that the set is self-contained. + --include_source_info When using --descriptor_set_out, do not strip + SourceCodeInfo from the FileDescriptorProto. + This results in vastly larger descriptors that + include information about the original + location of each decl in the source file as + well as surrounding comments. + --dependency_out=FILE Write a dependency output file in the format + expected by make. This writes the transitive + set of input file paths to FILE + --error_format=FORMAT Set the format in which to print errors. + FORMAT may be 'gcc' (the default) or 'msvs' + (Microsoft Visual Studio format). + --print_free_field_numbers Print the free field numbers of the messages + defined in the given proto files. Groups share + the same field number space with the parent + message. Extension ranges are counted as + occupied fields numbers. + + --plugin=EXECUTABLE Specifies a plugin executable to use. + Normally, protoc searches the PATH for + plugins, but you may specify additional + executables not in the path using this flag. + Additionally, EXECUTABLE may be of the form + NAME=PATH, in which case the given plugin name + is mapped to the given executable even if + the executable's own name differs. + --cpp_out=OUT_DIR Generate C++ header and source. + --csharp_out=OUT_DIR Generate C# source file. + --java_out=OUT_DIR Generate Java source file. + --javanano_out=OUT_DIR Generate Java Nano source file. + --js_out=OUT_DIR Generate JavaScript source. + --objc_out=OUT_DIR Generate Objective C header and source. + --php_out=OUT_DIR Generate PHP source file. + --python_out=OUT_DIR Generate Python source file. + --ruby_out=OUT_DIR Generate Ruby source file. + @ Read options and filenames from file. If a + relative file path is specified, the file + will be searched in the working directory. + The --proto_path option will not affect how + this argument file is searched. Content of + the file will be expanded in the position of + @ as in the argument list. Note + that shell expansion is not applied to the + content of the file (i.e., you cannot use + quotes, wildcards, escapes, commands, etc.). + Each line corresponds to a single argument, + even if it contains spaces. + */ static string[] s_supportedGenerators = new[] { "cpp", "csharp", "java", "javanano", "js", "objc", diff --git a/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs b/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs index 9afea9255e5..74aaa8bd3d5 100644 --- a/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs +++ b/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs @@ -68,7 +68,7 @@ namespace Grpc.Tools { var outputs = generator.GetPossibleOutputs(proto); foreach (string output in outputs) { var ti = new TaskItem(output); - ti.SetMetadata(Metadata.kSource, proto.ItemSpec); + ti.SetMetadata(Metadata.Source, proto.ItemSpec); possible.Add(ti); } } diff --git a/src/csharp/Grpc.Tools/ProtoReadDependencies.cs b/src/csharp/Grpc.Tools/ProtoReadDependencies.cs index 2ee03891469..ea931b08268 100644 --- a/src/csharp/Grpc.Tools/ProtoReadDependencies.cs +++ b/src/csharp/Grpc.Tools/ProtoReadDependencies.cs @@ -55,7 +55,7 @@ namespace Grpc.Tools { string[] deps = DepFileUtil.ReadDependencyInputs(ProtoDepDir, proto.ItemSpec, Log); foreach (string dep in deps) { var ti = new TaskItem(dep); - ti.SetMetadata(Metadata.kSource, proto.ItemSpec); + ti.SetMetadata(Metadata.Source, proto.ItemSpec); dependencies.Add(ti); } } diff --git a/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets index 5a8d3f20276..2ed515c8db7 100644 --- a/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets +++ b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.targets @@ -7,7 +7,7 @@ true <_Protobuf_MsBuildAssembly Condition=" '$(MSBuildRuntimeType)' == 'Core' ">netstandard1.3\Protobuf.MSBuild.dll - <_Protobuf_MsBuildAssembly Condition=" '$(MSBuildRuntimeType)' != 'Core' ">net40\Protobuf.MSBuild.dll + <_Protobuf_MsBuildAssembly Condition=" '$(MSBuildRuntimeType)' != 'Core' ">net45\Protobuf.MSBuild.dll @@ -370,7 +370,7 @@ Design-time support =================================================================================--> - Date: Mon, 13 Aug 2018 14:16:31 +0200 Subject: [PATCH 03/53] add linux/mac net45 compilation fix to Grcp.Tools.Tests.csproj --- src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index 661d97d81bc..7c1a6afe343 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -9,6 +9,18 @@ + + + + /usr/lib/mono/4.5-api + /usr/local/lib/mono/4.5-api + /Library/Frameworks/Mono.framework/Versions/Current/lib/mono/4.5-api + + From 67c2d99189620ea930aba4b8b3cbca18e21bc495 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 14:48:01 +0200 Subject: [PATCH 04/53] add Grpc.Tools.Tests to SanityTest --- src/csharp/Grpc.Core.Tests/SanityTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs index 73efad1f842..13c8c0d1431 100644 --- a/src/csharp/Grpc.Core.Tests/SanityTest.cs +++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs @@ -102,6 +102,7 @@ namespace Grpc.Core.Tests "Grpc.HealthCheck.Tests", "Grpc.IntegrationTesting", "Grpc.Reflection.Tests", + "Grpc.Tools.Tests", }; foreach (var assemblyName in otherAssemblies) { From 62901c1631e21657554a17dba9bfee550342edfd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 14:55:22 +0200 Subject: [PATCH 05/53] C# sanity test accepts [TestCase] attribute too --- src/csharp/Grpc.Core.Tests/SanityTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs index 13c8c0d1431..0904453b6e9 100644 --- a/src/csharp/Grpc.Core.Tests/SanityTest.cs +++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs @@ -65,13 +65,13 @@ namespace Grpc.Core.Tests { foreach (var m in t.GetMethods()) { - var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true); - if (attributes.Length > 0) + var testAttributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true); + var testCaseAttributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestCaseAttribute), true); + if (testAttributes.Length > 0 || testCaseAttributes.Length > 0) { testClasses.Add(t.FullName); break; } - } } testClasses.Sort(); From 7a9a9f6d1a79bdb4858b2eb0f41e9badc55ae8db Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 14:56:08 +0200 Subject: [PATCH 06/53] fix tests.json --- src/csharp/tests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/tests.json b/src/csharp/tests.json index 483d1d7aada..d06fbb7869a 100644 --- a/src/csharp/tests.json +++ b/src/csharp/tests.json @@ -66,11 +66,11 @@ "Grpc.Tools.Tests": [ "Grpc.Tools.Tests.CppGeneratorTests", "Grpc.Tools.Tests.CSharpGeneratorTests", + "Grpc.Tools.Tests.DepFileUtilTests", "Grpc.Tools.Tests.GeneratorTests", "Grpc.Tools.Tests.ProtoCompileBasicTests", "Grpc.Tools.Tests.ProtoCompileCommandLineGeneratorTests", "Grpc.Tools.Tests.ProtoCompileCommandLinePrinterTests", - "Grpc.Tools.Tests.ProtoToolsPlatformTaskTests", - "Grps.Tools.Tests.DepFileUtilTests" + "Grpc.Tools.Tests.ProtoToolsPlatformTaskTests" ] } From 86df93d4e1e4b3d8a6450bf9a19a77c8e7fbaee9 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 15:00:44 +0200 Subject: [PATCH 07/53] split up ProtoCompile tests to individual files --- .../ProtoCompileBasicTests.cs | 70 ++++++++++++++++++ ... ProtoCompileCommandLineGeneratorTests.cs} | 72 +------------------ .../ProtoCompileCommandLinePrinterTests.cs | 48 +++++++++++++ 3 files changed, 119 insertions(+), 71 deletions(-) create mode 100644 src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs rename src/csharp/Grpc.Tools.Tests/{ProtoCompileTaskTest.cs => ProtoCompileCommandLineGeneratorTests.cs} (68%) create mode 100644 src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs new file mode 100644 index 00000000000..0660b3e33d4 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs @@ -0,0 +1,70 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Reflection; +using Microsoft.Build.Framework; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + public class ProtoCompileBasicTests { + // Mock task class that stops right before invoking protoc. + public class ProtoCompileTestable : ProtoCompile { + public string LastPathToTool { get; private set; } + public string[] LastResponseFile { get; private set; } + + protected override int ExecuteTool(string pathToTool, + string response, + string commandLine) { + // We should never be using command line commands. + Assert.That(commandLine, Is.Null | Is.Empty); + + // Must receive a path to tool + Assert.That(pathToTool, Is.Not.Null & Is.Not.Empty); + Assert.That(response, Is.Not.Null & Does.EndWith("\n")); + + LastPathToTool = pathToTool; + LastResponseFile = response.Remove(response.Length - 1).Split('\n'); + + // Do not run the tool, but pretend it ran successfully. + return 0; + } + }; + + protected Mock _mockEngine; + protected ProtoCompileTestable _task; + + [SetUp] + public void SetUp() { + _mockEngine = new Mock(); + _task = new ProtoCompileTestable { + BuildEngine = _mockEngine.Object + }; + } + + [TestCase("ProtoBuf")] + [TestCase("Generator")] + [TestCase("OutputDir")] + [Description("We trust MSBuild to initialize these properties.")] + public void RequiredAttributePresentOnProperty(string prop) { + var pinfo = _task.GetType()?.GetProperty(prop); + Assert.NotNull(pinfo); + Assert.That(pinfo, Has.Attribute()); + } + } +} diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs similarity index 68% rename from src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs rename to src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs index 86c78289b29..fcdff11da45 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileTaskTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs @@ -22,52 +22,6 @@ using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - public class ProtoCompileBasicTests { - // Mock task class that stops right before invoking protoc. - public class ProtoCompileTestable : ProtoCompile { - public string LastPathToTool { get; private set; } - public string[] LastResponseFile { get; private set; } - - protected override int ExecuteTool(string pathToTool, - string response, - string commandLine) { - // We should never be using command line commands. - Assert.That(commandLine, Is.Null | Is.Empty); - - // Must receive a path to tool - Assert.That(pathToTool, Is.Not.Null & Is.Not.Empty); - Assert.That(response, Is.Not.Null & Does.EndWith("\n")); - - LastPathToTool = pathToTool; - LastResponseFile = response.Remove(response.Length - 1).Split('\n'); - - // Do not run the tool, but pretend it ran successfully. - return 0; - } - }; - - protected Mock _mockEngine; - protected ProtoCompileTestable _task; - - [SetUp] - public void SetUp() { - _mockEngine = new Mock(); - _task = new ProtoCompileTestable { - BuildEngine = _mockEngine.Object - }; - } - - [TestCase("ProtoBuf")] - [TestCase("Generator")] - [TestCase("OutputDir")] - [Description("We trust MSBuild to initialize these properties.")] - public void RequiredAttributePresentOnProperty(string prop) { - var pinfo = _task.GetType()?.GetProperty(prop); - Assert.NotNull(pinfo); - Assert.That(pinfo, Has.Attribute()); - } - }; - internal class ProtoCompileCommandLineGeneratorTests : ProtoCompileBasicTests { [SetUp] public new void SetUp() { @@ -204,29 +158,5 @@ namespace Grpc.Tools.Tests { Assert.That(_task.LastResponseFile, Does.Contain("--csharp_out=" + expect)); } - }; - - internal class ProtoCompileCommandLinePrinterTests : ProtoCompileBasicTests { - [SetUp] - public new void SetUp() { - _task.Generator = "csharp"; - _task.OutputDir = "outdir"; - _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); - - _mockEngine - .Setup(me => me.LogMessageEvent(It.IsAny())) - .Callback((BuildMessageEventArgs e) => - Assert.Fail($"Error logged by build engine:\n{e.Message}")) - .Verifiable("Command line was not output by the task."); - } - - void ExecuteExpectSuccess() { - _mockEngine - .Setup(me => me.LogErrorEvent(It.IsAny())) - .Callback((BuildErrorEventArgs e) => - Assert.Fail($"Error logged by build engine:\n{e.Message}")); - bool result = _task.Execute(); - Assert.IsTrue(result); - } - }; + } } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs new file mode 100644 index 00000000000..91cfceaf568 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs @@ -0,0 +1,48 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Reflection; +using Microsoft.Build.Framework; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + internal class ProtoCompileCommandLinePrinterTests : ProtoCompileBasicTests { + [SetUp] + public new void SetUp() { + _task.Generator = "csharp"; + _task.OutputDir = "outdir"; + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); + + _mockEngine + .Setup(me => me.LogMessageEvent(It.IsAny())) + .Callback((BuildMessageEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")) + .Verifiable("Command line was not output by the task."); + } + + void ExecuteExpectSuccess() { + _mockEngine + .Setup(me => me.LogErrorEvent(It.IsAny())) + .Callback((BuildErrorEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")); + bool result = _task.Execute(); + Assert.IsTrue(result); + } + } +} From 8a98ab97acaa831e9e4dc257ac4dbd0ef10b3203 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 15:01:31 +0200 Subject: [PATCH 08/53] fix typo in DepFileUtilTests --- src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs b/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs index 0ea621adea5..42d0ae5998b 100644 --- a/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs +++ b/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs @@ -23,7 +23,7 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using NUnit.Framework; -namespace Grps.Tools.Tests { +namespace Grpc.Tools.Tests { public class DepFileUtilTests { [Test] From 5f4dfaac9e8ad175cbd5ef074b08f087cc17a610 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 15:02:40 +0200 Subject: [PATCH 09/53] split up GeneratorTests into individual files --- .../Grpc.Tools.Tests/CSharpGeneratorTests.cs | 82 +++++++++++++ .../Grpc.Tools.Tests/CppGeneratorTests.cs | 83 +++++++++++++ src/csharp/Grpc.Tools.Tests/GeneratorTests.cs | 116 +----------------- 3 files changed, 166 insertions(+), 115 deletions(-) create mode 100644 src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs create mode 100644 src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs diff --git a/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs new file mode 100644 index 00000000000..dbffa65d8c8 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs @@ -0,0 +1,82 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + + public class CSharpGeneratorTests : GeneratorTests { + GeneratorServices _generator; + + [SetUp] + public new void SetUp() { + _generator = GeneratorServices.GetForLanguage("CSharp", _log); + } + + [TestCase("foo.proto", "Foo.cs", "FooGrpc.cs")] + [TestCase("sub/foo.proto", "Foo.cs", "FooGrpc.cs")] + [TestCase("one_two.proto", "OneTwo.cs", "OneTwoGrpc.cs")] + [TestCase("__one_two!.proto", "OneTwo!.cs", "OneTwo!Grpc.cs")] + [TestCase("one(two).proto", "One(two).cs", "One(two)Grpc.cs")] + [TestCase("one_(two).proto", "One(two).cs", "One(two)Grpc.cs")] + [TestCase("one two.proto", "One two.cs", "One twoGrpc.cs")] + [TestCase("one_ two.proto", "One two.cs", "One twoGrpc.cs")] + [TestCase("one .proto", "One .cs", "One Grpc.cs")] + public void NameMangling(string proto, string expectCs, string expectGrpcCs) { + var poss = _generator.GetPossibleOutputs(Utils.MakeItem(proto, "grpcservices", "both")); + Assert.AreEqual(2, poss.Length); + Assert.Contains(expectCs, poss); + Assert.Contains(expectGrpcCs, poss); + } + + [Test] + public void NoGrpcOneOutput() { + var poss = _generator.GetPossibleOutputs(Utils.MakeItem("foo.proto")); + Assert.AreEqual(1, poss.Length); + } + + [TestCase("none")] + [TestCase("")] + public void GrpcNoneOneOutput(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(1, poss.Length); + } + + [TestCase("client")] + [TestCase("server")] + [TestCase("both")] + public void GrpcEnabledTwoOutputs(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(2, poss.Length); + } + + [Test] + public void OutputDirMetadataRecognized() { + var item = Utils.MakeItem("foo.proto", "OutputDir", "out"); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(1, poss.Length); + Assert.That(poss[0], Is.EqualTo("out/Foo.cs") | Is.EqualTo("out\\Foo.cs")); + } + }; +} diff --git a/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs new file mode 100644 index 00000000000..e25a6498cd8 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs @@ -0,0 +1,83 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Moq; +using NUnit.Framework; + +namespace Grpc.Tools.Tests { + public class CppGeneratorTests : GeneratorTests { + GeneratorServices _generator; + + [SetUp] + public new void SetUp() { + _generator = GeneratorServices.GetForLanguage("Cpp", _log); + } + + [TestCase("foo.proto", "", "foo")] + [TestCase("foo.proto", ".", "foo")] + [TestCase("foo.proto", "./", "foo")] + [TestCase("sub/foo.proto", "", "sub/foo")] + [TestCase("root/sub/foo.proto", "root", "sub/foo")] + [TestCase("root/sub/foo.proto", "root", "sub/foo")] + [TestCase("/root/sub/foo.proto", "/root", "sub/foo")] + public void RelativeDirectoryCompute(string proto, string root, string expectStem) { + if (Path.DirectorySeparatorChar == '\\') + expectStem = expectStem.Replace('/', '\\'); + var poss = _generator.GetPossibleOutputs(Utils.MakeItem(proto, "ProtoRoot", root)); + Assert.AreEqual(2, poss.Length); + Assert.Contains(expectStem + ".pb.cc", poss); + Assert.Contains(expectStem + ".pb.h", poss); + } + + [Test] + public void NoGrpcTwoOutputs() { + var poss = _generator.GetPossibleOutputs(Utils.MakeItem("foo.proto")); + Assert.AreEqual(2, poss.Length); + } + + [TestCase("false")] + [TestCase("")] + public void GrpcDisabledTwoOutput(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(2, poss.Length); + } + + [TestCase("true")] + public void GrpcEnabledFourOutputs(string grpc) { + var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(4, poss.Length); + Assert.Contains("foo.pb.cc", poss); + Assert.Contains("foo.pb.h", poss); + Assert.Contains("foo_grpc.pb.cc", poss); + Assert.Contains("foo_grpc.pb.h", poss); + } + + [Test] + public void OutputDirMetadataRecognized() { + var item = Utils.MakeItem("foo.proto", "OutputDir", "out"); + var poss = _generator.GetPossibleOutputs(item); + Assert.AreEqual(2, poss.Length); + Assert.That(Path.GetDirectoryName(poss[0]), Is.EqualTo("out")); + } + } +} diff --git a/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs index 0a273380b98..93df9d6cab2 100644 --- a/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs +++ b/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs @@ -47,119 +47,5 @@ namespace Grpc.Tools.Tests { Assert.IsNull(GeneratorServices.GetForLanguage(lang, _log)); _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Once); } - }; - - public class CSharpGeneratorTests : GeneratorTests { - GeneratorServices _generator; - - [SetUp] - public new void SetUp() { - _generator = GeneratorServices.GetForLanguage("CSharp", _log); - } - - [TestCase("foo.proto", "Foo.cs", "FooGrpc.cs")] - [TestCase("sub/foo.proto", "Foo.cs", "FooGrpc.cs")] - [TestCase("one_two.proto", "OneTwo.cs", "OneTwoGrpc.cs")] - [TestCase("__one_two!.proto", "OneTwo!.cs", "OneTwo!Grpc.cs")] - [TestCase("one(two).proto", "One(two).cs", "One(two)Grpc.cs")] - [TestCase("one_(two).proto", "One(two).cs", "One(two)Grpc.cs")] - [TestCase("one two.proto", "One two.cs", "One twoGrpc.cs")] - [TestCase("one_ two.proto", "One two.cs", "One twoGrpc.cs")] - [TestCase("one .proto", "One .cs", "One Grpc.cs")] - public void NameMangling(string proto, string expectCs, string expectGrpcCs) { - var poss = _generator.GetPossibleOutputs(Utils.MakeItem(proto, "grpcservices", "both")); - Assert.AreEqual(2, poss.Length); - Assert.Contains(expectCs, poss); - Assert.Contains(expectGrpcCs, poss); - } - - [Test] - public void NoGrpcOneOutput() { - var poss = _generator.GetPossibleOutputs(Utils.MakeItem("foo.proto")); - Assert.AreEqual(1, poss.Length); - } - - [TestCase("none")] - [TestCase("")] - public void GrpcNoneOneOutput(string grpc) { - var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); - var poss = _generator.GetPossibleOutputs(item); - Assert.AreEqual(1, poss.Length); - } - - [TestCase("client")] - [TestCase("server")] - [TestCase("both")] - public void GrpcEnabledTwoOutputs(string grpc) { - var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); - var poss = _generator.GetPossibleOutputs(item); - Assert.AreEqual(2, poss.Length); - } - - [Test] - public void OutputDirMetadataRecognized() { - var item = Utils.MakeItem("foo.proto", "OutputDir", "out"); - var poss = _generator.GetPossibleOutputs(item); - Assert.AreEqual(1, poss.Length); - Assert.That(poss[0], Is.EqualTo("out/Foo.cs") | Is.EqualTo("out\\Foo.cs")); - } - }; - - public class CppGeneratorTests : GeneratorTests { - GeneratorServices _generator; - - [SetUp] - public new void SetUp() { - _generator = GeneratorServices.GetForLanguage("Cpp", _log); - } - - [TestCase("foo.proto", "", "foo")] - [TestCase("foo.proto", ".", "foo")] - [TestCase("foo.proto", "./", "foo")] - [TestCase("sub/foo.proto", "", "sub/foo")] - [TestCase("root/sub/foo.proto", "root", "sub/foo")] - [TestCase("root/sub/foo.proto", "root", "sub/foo")] - [TestCase("/root/sub/foo.proto", "/root", "sub/foo")] - public void RelativeDirectoryCompute(string proto, string root, string expectStem) { - if (Path.DirectorySeparatorChar == '\\') - expectStem = expectStem.Replace('/', '\\'); - var poss = _generator.GetPossibleOutputs(Utils.MakeItem(proto, "ProtoRoot", root)); - Assert.AreEqual(2, poss.Length); - Assert.Contains(expectStem + ".pb.cc", poss); - Assert.Contains(expectStem + ".pb.h", poss); - } - - [Test] - public void NoGrpcTwoOutputs() { - var poss = _generator.GetPossibleOutputs(Utils.MakeItem("foo.proto")); - Assert.AreEqual(2, poss.Length); - } - - [TestCase("false")] - [TestCase("")] - public void GrpcDisabledTwoOutput(string grpc) { - var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); - var poss = _generator.GetPossibleOutputs(item); - Assert.AreEqual(2, poss.Length); - } - - [TestCase("true")] - public void GrpcEnabledFourOutputs(string grpc) { - var item = Utils.MakeItem("foo.proto", "grpcservices", grpc); - var poss = _generator.GetPossibleOutputs(item); - Assert.AreEqual(4, poss.Length); - Assert.Contains("foo.pb.cc", poss); - Assert.Contains("foo.pb.h", poss); - Assert.Contains("foo_grpc.pb.cc", poss); - Assert.Contains("foo_grpc.pb.h", poss); - } - - [Test] - public void OutputDirMetadataRecognized() { - var item = Utils.MakeItem("foo.proto", "OutputDir", "out"); - var poss = _generator.GetPossibleOutputs(item); - Assert.AreEqual(2, poss.Length); - Assert.That(Path.GetDirectoryName(poss[0]), Is.EqualTo("out")); - } - }; + } } From e3e7e32a7e1698c7937e5bf0bde4914e13131362 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 13 Aug 2018 15:55:54 +0200 Subject: [PATCH 10/53] sync nunit version for all test projects --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 4 ++-- src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj | 4 ++-- .../Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj | 4 ++-- .../Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 4 ++-- src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj | 4 ++-- src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 3 ++- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 18993a93e00..d58f0468244 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -17,8 +17,8 @@ - - + + diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index d2cc5bbc65e..7493eb8051d 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -17,8 +17,8 @@ - - + + diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index 9da0539dcb9..616e56df105 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -16,8 +16,8 @@ - - + + diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index e4f36d8810d..ad7033b7829 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -19,8 +19,8 @@ - - + + diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index d3686971245..0c12f38f25a 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -16,8 +16,8 @@ - - + + diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index 7c1a6afe343..ab14f32afd4 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -27,7 +27,8 @@ - + + From 4a1a447d35a705c42d6b5d70851a22e7fd67b466 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Thu, 16 Aug 2018 15:12:45 +0800 Subject: [PATCH 11/53] Formalize the contract of grpc_call_start_batch with empty batches --- include/grpc/grpc.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 897b89851a0..546243623f3 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -241,10 +241,13 @@ GRPCAPI void* grpc_call_arena_alloc(grpc_call* call, size_t size); appropriate to call grpc_completion_queue_next or grpc_completion_queue_pluck consequent to the failed grpc_call_start_batch call. + If a call to grpc_call_start_batch with an empty batch returns + GRPC_CALL_OK, the tag is put in the completion queue immediately. THREAD SAFETY: access to grpc_call_start_batch in multi-threaded environment needs to be synchronized. As an optimization, you may synchronize batches containing just send operations independently from batches containing just - receive operations. */ + receive operations. Access to grpc_call_start_batch with an empty batch is + thread-compatible. */ GRPCAPI grpc_call_error grpc_call_start_batch(grpc_call* call, const grpc_op* ops, size_t nops, void* tag, void* reserved); From be1ce0c4ccbf17ebeee9b7b057d40ff4e12f3479 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 25 Sep 2018 11:42:57 -0700 Subject: [PATCH 12/53] Revert "Merge pull request #16600 from grpc/revert-16574-health_checking_service" This reverts commit de11358660b6474ffea480d8d756c23e58a7cf07, reversing changes made to 107d10ea73f77dc9bb498c9b91e1fcd0188dfb45. --- .../grpcpp/impl/codegen/completion_queue.h | 1 + .../health/default_health_check_service.cc | 484 +++++++++++++++--- .../health/default_health_check_service.h | 242 ++++++++- src/cpp/server/health/health.pb.c | 1 - src/cpp/server/health/health.pb.h | 7 +- src/cpp/server/server_cc.cc | 27 +- src/proto/grpc/health/v1/health.proto | 20 + .../end2end/health_service_end2end_test.cc | 76 ++- tools/distrib/check_nanopb_output.sh | 18 + 9 files changed, 771 insertions(+), 105 deletions(-) diff --git a/include/grpcpp/impl/codegen/completion_queue.h b/include/grpcpp/impl/codegen/completion_queue.h index f52f9a53bea..a62d48abe7e 100644 --- a/include/grpcpp/impl/codegen/completion_queue.h +++ b/include/grpcpp/impl/codegen/completion_queue.h @@ -387,6 +387,7 @@ class ServerCompletionQueue : public CompletionQueue { grpc_cq_polling_type polling_type_; friend class ServerBuilder; + friend class Server; }; } // namespace grpc diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index bfda67d0864..fc3db1bba73 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -30,29 +30,162 @@ #include "src/cpp/server/health/health.pb.h" namespace grpc { + +// +// DefaultHealthCheckService +// + +DefaultHealthCheckService::DefaultHealthCheckService() { + services_map_[""].SetServingStatus(SERVING); +} + +void DefaultHealthCheckService::SetServingStatus( + const grpc::string& service_name, bool serving) { + std::unique_lock lock(mu_); + services_map_[service_name].SetServingStatus(serving ? SERVING : NOT_SERVING); +} + +void DefaultHealthCheckService::SetServingStatus(bool serving) { + const ServingStatus status = serving ? SERVING : NOT_SERVING; + std::unique_lock lock(mu_); + for (auto& p : services_map_) { + ServiceData& service_data = p.second; + service_data.SetServingStatus(status); + } +} + +DefaultHealthCheckService::ServingStatus +DefaultHealthCheckService::GetServingStatus( + const grpc::string& service_name) const { + std::lock_guard lock(mu_); + auto it = services_map_.find(service_name); + if (it == services_map_.end()) { + return NOT_FOUND; + } + const ServiceData& service_data = it->second; + return service_data.GetServingStatus(); +} + +void DefaultHealthCheckService::RegisterCallHandler( + const grpc::string& service_name, + std::shared_ptr handler) { + std::unique_lock lock(mu_); + ServiceData& service_data = services_map_[service_name]; + service_data.AddCallHandler(handler /* copies ref */); + handler->SendHealth(std::move(handler), service_data.GetServingStatus()); +} + +void DefaultHealthCheckService::UnregisterCallHandler( + const grpc::string& service_name, + std::shared_ptr handler) { + std::unique_lock lock(mu_); + auto it = services_map_.find(service_name); + if (it == services_map_.end()) return; + ServiceData& service_data = it->second; + service_data.RemoveCallHandler(std::move(handler)); + if (service_data.Unused()) { + services_map_.erase(it); + } +} + +DefaultHealthCheckService::HealthCheckServiceImpl* +DefaultHealthCheckService::GetHealthCheckService( + std::unique_ptr cq) { + GPR_ASSERT(impl_ == nullptr); + impl_.reset(new HealthCheckServiceImpl(this, std::move(cq))); + return impl_.get(); +} + +// +// DefaultHealthCheckService::ServiceData +// + +void DefaultHealthCheckService::ServiceData::SetServingStatus( + ServingStatus status) { + status_ = status; + for (auto& call_handler : call_handlers_) { + call_handler->SendHealth(call_handler /* copies ref */, status); + } +} + +void DefaultHealthCheckService::ServiceData::AddCallHandler( + std::shared_ptr handler) { + call_handlers_.insert(std::move(handler)); +} + +void DefaultHealthCheckService::ServiceData::RemoveCallHandler( + std::shared_ptr handler) { + call_handlers_.erase(std::move(handler)); +} + +// +// DefaultHealthCheckService::HealthCheckServiceImpl +// + namespace { const char kHealthCheckMethodName[] = "/grpc.health.v1.Health/Check"; +const char kHealthWatchMethodName[] = "/grpc.health.v1.Health/Watch"; } // namespace DefaultHealthCheckService::HealthCheckServiceImpl::HealthCheckServiceImpl( - DefaultHealthCheckService* service) - : service_(service), method_(nullptr) { - internal::MethodHandler* handler = - new internal::RpcMethodHandler( - std::mem_fn(&HealthCheckServiceImpl::Check), this); - method_ = new internal::RpcServiceMethod( - kHealthCheckMethodName, internal::RpcMethod::NORMAL_RPC, handler); - AddMethod(method_); -} - -Status DefaultHealthCheckService::HealthCheckServiceImpl::Check( - ServerContext* context, const ByteBuffer* request, ByteBuffer* response) { - // Decode request. - std::vector slices; - if (!request->Dump(&slices).ok()) { - return Status(StatusCode::INVALID_ARGUMENT, ""); + DefaultHealthCheckService* database, + std::unique_ptr cq) + : database_(database), cq_(std::move(cq)) { + // Add Check() method. + check_method_ = new internal::RpcServiceMethod( + kHealthCheckMethodName, internal::RpcMethod::NORMAL_RPC, nullptr); + AddMethod(check_method_); + // Add Watch() method. + watch_method_ = new internal::RpcServiceMethod( + kHealthWatchMethodName, internal::RpcMethod::SERVER_STREAMING, nullptr); + AddMethod(watch_method_); + // Create serving thread. + thread_ = std::unique_ptr<::grpc_core::Thread>( + new ::grpc_core::Thread("grpc_health_check_service", Serve, this)); +} + +DefaultHealthCheckService::HealthCheckServiceImpl::~HealthCheckServiceImpl() { + // We will reach here after the server starts shutting down. + shutdown_ = true; + { + std::unique_lock lock(cq_shutdown_mu_); + cq_->Shutdown(); + } + thread_->Join(); +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::StartServingThread() { + thread_->Start(); +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::Serve(void* arg) { + HealthCheckServiceImpl* service = + reinterpret_cast(arg); + // TODO(juanlishen): This is a workaround to wait for the cq to be ready. + // Need to figure out why cq is not ready after service starts. + gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(1, GPR_TIMESPAN))); + CheckCallHandler::CreateAndStart(service->cq_.get(), service->database_, + service); + WatchCallHandler::CreateAndStart(service->cq_.get(), service->database_, + service); + void* tag; + bool ok; + while (true) { + if (!service->cq_->Next(&tag, &ok)) { + // The completion queue is shutting down. + GPR_ASSERT(service->shutdown_); + break; + } + auto* next_step = static_cast(tag); + next_step->Run(ok); } +} + +bool DefaultHealthCheckService::HealthCheckServiceImpl::DecodeRequest( + const ByteBuffer& request, grpc::string* service_name) { + std::vector slices; + if (!request.Dump(&slices).ok()) return false; uint8_t* request_bytes = nullptr; bool request_bytes_owned = false; size_t request_size = 0; @@ -64,14 +197,13 @@ Status DefaultHealthCheckService::HealthCheckServiceImpl::Check( request_size = slices[0].size(); } else { request_bytes_owned = true; - request_bytes = static_cast(gpr_malloc(request->Length())); + request_bytes = static_cast(gpr_malloc(request.Length())); uint8_t* copy_to = request_bytes; for (size_t i = 0; i < slices.size(); i++) { memcpy(copy_to, slices[i].begin(), slices[i].size()); copy_to += slices[i].size(); } } - if (request_bytes != nullptr) { pb_istream_t istream = pb_istream_from_buffer(request_bytes, request_size); bool decode_status = pb_decode( @@ -79,26 +211,22 @@ Status DefaultHealthCheckService::HealthCheckServiceImpl::Check( if (request_bytes_owned) { gpr_free(request_bytes); } - if (!decode_status) { - return Status(StatusCode::INVALID_ARGUMENT, ""); - } - } - - // Check status from the associated default health checking service. - DefaultHealthCheckService::ServingStatus serving_status = - service_->GetServingStatus( - request_struct.has_service ? request_struct.service : ""); - if (serving_status == DefaultHealthCheckService::NOT_FOUND) { - return Status(StatusCode::NOT_FOUND, ""); + if (!decode_status) return false; } + *service_name = request_struct.has_service ? request_struct.service : ""; + return true; +} - // Encode response +bool DefaultHealthCheckService::HealthCheckServiceImpl::EncodeResponse( + ServingStatus status, ByteBuffer* response) { grpc_health_v1_HealthCheckResponse response_struct; response_struct.has_status = true; response_struct.status = - serving_status == DefaultHealthCheckService::SERVING - ? grpc_health_v1_HealthCheckResponse_ServingStatus_SERVING - : grpc_health_v1_HealthCheckResponse_ServingStatus_NOT_SERVING; + status == NOT_FOUND + ? grpc_health_v1_HealthCheckResponse_ServingStatus_SERVICE_UNKNOWN + : status == SERVING + ? grpc_health_v1_HealthCheckResponse_ServingStatus_SERVING + : grpc_health_v1_HealthCheckResponse_ServingStatus_NOT_SERVING; pb_ostream_t ostream; memset(&ostream, 0, sizeof(ostream)); pb_encode(&ostream, grpc_health_v1_HealthCheckResponse_fields, @@ -108,48 +236,282 @@ Status DefaultHealthCheckService::HealthCheckServiceImpl::Check( GRPC_SLICE_LENGTH(response_slice)); bool encode_status = pb_encode( &ostream, grpc_health_v1_HealthCheckResponse_fields, &response_struct); - if (!encode_status) { - return Status(StatusCode::INTERNAL, "Failed to encode response."); - } + if (!encode_status) return false; Slice encoded_response(response_slice, Slice::STEAL_REF); ByteBuffer response_buffer(&encoded_response, 1); response->Swap(&response_buffer); - return Status::OK; + return true; } -DefaultHealthCheckService::DefaultHealthCheckService() { - services_map_.emplace("", true); +// +// DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler +// + +void DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: + CreateAndStart(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service) { + std::shared_ptr self = + std::make_shared(cq, database, service); + CheckCallHandler* handler = static_cast(self.get()); + { + std::unique_lock lock(service->cq_shutdown_mu_); + if (service->shutdown_) return; + // Request a Check() call. + handler->next_ = + CallableTag(std::bind(&CheckCallHandler::OnCallReceived, handler, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + service->RequestAsyncUnary(0, &handler->ctx_, &handler->request_, + &handler->writer_, cq, cq, &handler->next_); + } } -void DefaultHealthCheckService::SetServingStatus( - const grpc::string& service_name, bool serving) { - std::lock_guard lock(mu_); - services_map_[service_name] = serving; +DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: + CheckCallHandler(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service) + : cq_(cq), database_(database), service_(service), writer_(&ctx_) {} + +void DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: + OnCallReceived(std::shared_ptr self, bool ok) { + if (!ok) { + // The value of ok being false means that the server is shutting down. + return; + } + // Spawn a new handler instance to serve the next new client. Every handler + // instance will deallocate itself when it's done. + CreateAndStart(cq_, database_, service_); + // Process request. + gpr_log(GPR_DEBUG, "[HCS %p] Health check started for handler %p", service_, + this); + grpc::string service_name; + grpc::Status status = Status::OK; + ByteBuffer response; + if (!service_->DecodeRequest(request_, &service_name)) { + status = Status(StatusCode::INVALID_ARGUMENT, ""); + } else { + ServingStatus serving_status = database_->GetServingStatus(service_name); + if (serving_status == NOT_FOUND) { + status = Status(StatusCode::NOT_FOUND, "service name unknown"); + } else if (!service_->EncodeResponse(serving_status, &response)) { + status = Status(StatusCode::INTERNAL, ""); + } + } + // Send response. + { + std::unique_lock lock(service_->cq_shutdown_mu_); + if (!service_->shutdown_) { + next_ = + CallableTag(std::bind(&CheckCallHandler::OnFinishDone, this, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + if (status.ok()) { + writer_.Finish(response, status, &next_); + } else { + writer_.FinishWithError(status, &next_); + } + } + } } -void DefaultHealthCheckService::SetServingStatus(bool serving) { - std::lock_guard lock(mu_); - for (auto iter = services_map_.begin(); iter != services_map_.end(); ++iter) { - iter->second = serving; +void DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: + OnFinishDone(std::shared_ptr self, bool ok) { + if (ok) { + gpr_log(GPR_DEBUG, "[HCS %p] Health check call finished for handler %p", + service_, this); } } -DefaultHealthCheckService::ServingStatus -DefaultHealthCheckService::GetServingStatus( - const grpc::string& service_name) const { - std::lock_guard lock(mu_); - const auto& iter = services_map_.find(service_name); - if (iter == services_map_.end()) { - return NOT_FOUND; +// +// DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler +// + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + CreateAndStart(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service) { + std::shared_ptr self = + std::make_shared(cq, database, service); + WatchCallHandler* handler = static_cast(self.get()); + { + std::unique_lock lock(service->cq_shutdown_mu_); + if (service->shutdown_) return; + // Request AsyncNotifyWhenDone(). + handler->on_done_notified_ = + CallableTag(std::bind(&WatchCallHandler::OnDoneNotified, handler, + std::placeholders::_1, std::placeholders::_2), + self /* copies ref */); + handler->ctx_.AsyncNotifyWhenDone(&handler->on_done_notified_); + // Request a Watch() call. + handler->next_ = + CallableTag(std::bind(&WatchCallHandler::OnCallReceived, handler, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + service->RequestAsyncServerStreaming(1, &handler->ctx_, &handler->request_, + &handler->stream_, cq, cq, + &handler->next_); } - return iter->second ? SERVING : NOT_SERVING; } -DefaultHealthCheckService::HealthCheckServiceImpl* -DefaultHealthCheckService::GetHealthCheckService() { - GPR_ASSERT(impl_ == nullptr); - impl_.reset(new HealthCheckServiceImpl(this)); - return impl_.get(); +DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + WatchCallHandler(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service) + : cq_(cq), + database_(database), + service_(service), + stream_(&ctx_), + call_state_(WAITING_FOR_CALL) {} + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + OnCallReceived(std::shared_ptr self, bool ok) { + if (ok) { + call_state_ = CALL_RECEIVED; + } else { + // AsyncNotifyWhenDone() needs to be called before the call starts, but the + // tag will not pop out if the call never starts ( + // https://github.com/grpc/grpc/issues/10136). So we need to manually + // release the ownership of the handler in this case. + GPR_ASSERT(on_done_notified_.ReleaseHandler() != nullptr); + } + if (!ok || shutdown_) { + // The value of ok being false means that the server is shutting down. + Shutdown(std::move(self), "OnCallReceived"); + return; + } + // Spawn a new handler instance to serve the next new client. Every handler + // instance will deallocate itself when it's done. + CreateAndStart(cq_, database_, service_); + // Parse request. + if (!service_->DecodeRequest(request_, &service_name_)) { + on_finish_done_ = + CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + stream_.Finish(Status(StatusCode::INVALID_ARGUMENT, ""), &on_finish_done_); + call_state_ = FINISH_CALLED; + return; + } + // Register the call for updates to the service. + gpr_log(GPR_DEBUG, + "[HCS %p] Health check watch started for service \"%s\" " + "(handler: %p)", + service_, service_name_.c_str(), this); + database_->RegisterCallHandler(service_name_, std::move(self)); +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + SendHealth(std::shared_ptr self, ServingStatus status) { + std::unique_lock lock(mu_); + // If there's already a send in flight, cache the new status, and + // we'll start a new send for it when the one in flight completes. + if (send_in_flight_) { + pending_status_ = status; + return; + } + // Start a send. + SendHealthLocked(std::move(self), status); +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + SendHealthLocked(std::shared_ptr self, ServingStatus status) { + std::unique_lock cq_lock(service_->cq_shutdown_mu_); + if (service_->shutdown_) { + cq_lock.release()->unlock(); + Shutdown(std::move(self), "SendHealthLocked"); + return; + } + send_in_flight_ = true; + call_state_ = SEND_MESSAGE_PENDING; + // Construct response. + ByteBuffer response; + if (!service_->EncodeResponse(status, &response)) { + on_finish_done_ = + CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + stream_.Finish(Status(StatusCode::INTERNAL, ""), &on_finish_done_); + return; + } + next_ = CallableTag(std::bind(&WatchCallHandler::OnSendHealthDone, this, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + stream_.Write(response, &next_); +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + OnSendHealthDone(std::shared_ptr self, bool ok) { + if (!ok || shutdown_) { + Shutdown(std::move(self), "OnSendHealthDone"); + return; + } + call_state_ = CALL_RECEIVED; + { + std::unique_lock lock(mu_); + send_in_flight_ = false; + // If we got a new status since we started the last send, start a + // new send for it. + if (pending_status_ != NOT_FOUND) { + auto status = pending_status_; + pending_status_ = NOT_FOUND; + SendHealthLocked(std::move(self), status); + } + } +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + OnDoneNotified(std::shared_ptr self, bool ok) { + GPR_ASSERT(ok); + done_notified_ = true; + if (ctx_.IsCancelled()) { + is_cancelled_ = true; + } + gpr_log(GPR_DEBUG, + "[HCS %p] Healt check call is notified done (handler: %p, " + "is_cancelled: %d).", + service_, this, static_cast(is_cancelled_)); + Shutdown(std::move(self), "OnDoneNotified"); +} + +// TODO(roth): This method currently assumes that there will be only one +// thread polling the cq and invoking the corresponding callbacks. If +// that changes, we will need to add synchronization here. +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + Shutdown(std::shared_ptr self, const char* reason) { + if (!shutdown_) { + gpr_log(GPR_DEBUG, + "[HCS %p] Shutting down the handler (service_name: \"%s\", " + "handler: %p, reason: %s).", + service_, service_name_.c_str(), this, reason); + shutdown_ = true; + } + // OnCallReceived() may be called after OnDoneNotified(), so we need to + // try to Finish() every time we are in Shutdown(). + if (call_state_ >= CALL_RECEIVED && call_state_ < FINISH_CALLED) { + std::unique_lock lock(service_->cq_shutdown_mu_); + if (!service_->shutdown_) { + on_finish_done_ = + CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + // TODO(juanlishen): Maybe add a message proto for the client to + // explicitly cancel the stream so that we can return OK status in such + // cases. + stream_.Finish(Status::CANCELLED, &on_finish_done_); + call_state_ = FINISH_CALLED; + } + } +} + +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + OnFinishDone(std::shared_ptr self, bool ok) { + if (ok) { + gpr_log(GPR_DEBUG, + "[HCS %p] Health check call finished (service_name: \"%s\", " + "handler: %p).", + service_, service_name_.c_str(), this); + } } } // namespace grpc diff --git a/src/cpp/server/health/default_health_check_service.h b/src/cpp/server/health/default_health_check_service.h index a1ce5aa64ec..edad5949362 100644 --- a/src/cpp/server/health/default_health_check_service.h +++ b/src/cpp/server/health/default_health_check_service.h @@ -19,42 +19,268 @@ #ifndef GRPC_INTERNAL_CPP_SERVER_DEFAULT_HEALTH_CHECK_SERVICE_H #define GRPC_INTERNAL_CPP_SERVER_DEFAULT_HEALTH_CHECK_SERVICE_H +#include #include +#include +#include +#include #include +#include +#include #include #include +#include "src/core/lib/gprpp/thd.h" + namespace grpc { // Default implementation of HealthCheckServiceInterface. Server will create and // own it. class DefaultHealthCheckService final : public HealthCheckServiceInterface { public: + enum ServingStatus { NOT_FOUND, SERVING, NOT_SERVING }; + // The service impl to register with the server. class HealthCheckServiceImpl : public Service { public: - explicit HealthCheckServiceImpl(DefaultHealthCheckService* service); + // Base class for call handlers. + class CallHandler { + public: + virtual ~CallHandler() = default; + virtual void SendHealth(std::shared_ptr self, + ServingStatus status) = 0; + }; - Status Check(ServerContext* context, const ByteBuffer* request, - ByteBuffer* response); + HealthCheckServiceImpl(DefaultHealthCheckService* database, + std::unique_ptr cq); + + ~HealthCheckServiceImpl(); + + void StartServingThread(); private: - const DefaultHealthCheckService* const service_; - internal::RpcServiceMethod* method_; + // A tag that can be called with a bool argument. It's tailored for + // CallHandler's use. Before being used, it should be constructed with a + // method of CallHandler and a shared pointer to the handler. The + // shared pointer will be moved to the invoked function and the function + // can only be invoked once. That makes ref counting of the handler easier, + // because the shared pointer is not bound to the function and can be gone + // once the invoked function returns (if not used any more). + class CallableTag { + public: + using HandlerFunction = + std::function, bool)>; + + CallableTag() {} + + CallableTag(HandlerFunction func, std::shared_ptr handler) + : handler_function_(std::move(func)), handler_(std::move(handler)) { + GPR_ASSERT(handler_function_ != nullptr); + GPR_ASSERT(handler_ != nullptr); + } + + // Runs the tag. This should be called only once. The handler is no + // longer owned by this tag after this method is invoked. + void Run(bool ok) { + GPR_ASSERT(handler_function_ != nullptr); + GPR_ASSERT(handler_ != nullptr); + handler_function_(std::move(handler_), ok); + } + + // Releases and returns the shared pointer to the handler. + std::shared_ptr ReleaseHandler() { + return std::move(handler_); + } + + private: + HandlerFunction handler_function_ = nullptr; + std::shared_ptr handler_; + }; + + // Call handler for Check method. + // Each handler takes care of one call. It contains per-call data and it + // will access the members of the parent class (i.e., + // DefaultHealthCheckService) for per-service health data. + class CheckCallHandler : public CallHandler { + public: + // Instantiates a CheckCallHandler and requests the next health check + // call. The handler object will manage its own lifetime, so no action is + // needed from the caller any more regarding that object. + static void CreateAndStart(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service); + + // This ctor is public because we want to use std::make_shared<> in + // CreateAndStart(). This ctor shouldn't be used elsewhere. + CheckCallHandler(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service); + + // Not used for Check. + void SendHealth(std::shared_ptr self, + ServingStatus status) override {} + + private: + // Called when we receive a call. + // Spawns a new handler so that we can keep servicing future calls. + void OnCallReceived(std::shared_ptr self, bool ok); + + // Called when Finish() is done. + void OnFinishDone(std::shared_ptr self, bool ok); + + // The members passed down from HealthCheckServiceImpl. + ServerCompletionQueue* cq_; + DefaultHealthCheckService* database_; + HealthCheckServiceImpl* service_; + + ByteBuffer request_; + GenericServerAsyncResponseWriter writer_; + ServerContext ctx_; + + CallableTag next_; + }; + + // Call handler for Watch method. + // Each handler takes care of one call. It contains per-call data and it + // will access the members of the parent class (i.e., + // DefaultHealthCheckService) for per-service health data. + class WatchCallHandler : public CallHandler { + public: + // Instantiates a WatchCallHandler and requests the next health check + // call. The handler object will manage its own lifetime, so no action is + // needed from the caller any more regarding that object. + static void CreateAndStart(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service); + + // This ctor is public because we want to use std::make_shared<> in + // CreateAndStart(). This ctor shouldn't be used elsewhere. + WatchCallHandler(ServerCompletionQueue* cq, + DefaultHealthCheckService* database, + HealthCheckServiceImpl* service); + + void SendHealth(std::shared_ptr self, + ServingStatus status) override; + + private: + // Called when we receive a call. + // Spawns a new handler so that we can keep servicing future calls. + void OnCallReceived(std::shared_ptr self, bool ok); + + // Requires holding mu_. + void SendHealthLocked(std::shared_ptr self, + ServingStatus status); + + // When sending a health result finishes. + void OnSendHealthDone(std::shared_ptr self, bool ok); + + // Called when Finish() is done. + void OnFinishDone(std::shared_ptr self, bool ok); + + // Called when AsyncNotifyWhenDone() notifies us. + void OnDoneNotified(std::shared_ptr self, bool ok); + + void Shutdown(std::shared_ptr self, const char* reason); + + // The members passed down from HealthCheckServiceImpl. + ServerCompletionQueue* cq_; + DefaultHealthCheckService* database_; + HealthCheckServiceImpl* service_; + + ByteBuffer request_; + grpc::string service_name_; + GenericServerAsyncWriter stream_; + ServerContext ctx_; + + std::mutex mu_; + bool send_in_flight_ = false; // Guarded by mu_. + ServingStatus pending_status_ = NOT_FOUND; // Guarded by mu_. + + // The state of the RPC progress. + enum CallState { + WAITING_FOR_CALL, + CALL_RECEIVED, + SEND_MESSAGE_PENDING, + FINISH_CALLED + } call_state_; + + bool shutdown_ = false; + bool done_notified_ = false; + bool is_cancelled_ = false; + CallableTag next_; + CallableTag on_done_notified_; + CallableTag on_finish_done_; + }; + + // Handles the incoming requests and drives the completion queue in a loop. + static void Serve(void* arg); + + // Returns true on success. + static bool DecodeRequest(const ByteBuffer& request, + grpc::string* service_name); + static bool EncodeResponse(ServingStatus status, ByteBuffer* response); + + // Needed to appease Windows compilers, which don't seem to allow + // nested classes to access protected members in the parent's + // superclass. + using Service::RequestAsyncServerStreaming; + using Service::RequestAsyncUnary; + + DefaultHealthCheckService* database_; + std::unique_ptr cq_; + internal::RpcServiceMethod* check_method_; + internal::RpcServiceMethod* watch_method_; + + // To synchronize the operations related to shutdown state of cq_, so that + // we don't enqueue new tags into cq_ after it is already shut down. + std::mutex cq_shutdown_mu_; + std::atomic_bool shutdown_{false}; + std::unique_ptr<::grpc_core::Thread> thread_; }; DefaultHealthCheckService(); + void SetServingStatus(const grpc::string& service_name, bool serving) override; void SetServingStatus(bool serving) override; - enum ServingStatus { NOT_FOUND, SERVING, NOT_SERVING }; + ServingStatus GetServingStatus(const grpc::string& service_name) const; - HealthCheckServiceImpl* GetHealthCheckService(); + + HealthCheckServiceImpl* GetHealthCheckService( + std::unique_ptr cq); private: + // Stores the current serving status of a service and any call + // handlers registered for updates when the service's status changes. + class ServiceData { + public: + void SetServingStatus(ServingStatus status); + ServingStatus GetServingStatus() const { return status_; } + void AddCallHandler( + std::shared_ptr handler); + void RemoveCallHandler( + std::shared_ptr handler); + bool Unused() const { + return call_handlers_.empty() && status_ == NOT_FOUND; + } + + private: + ServingStatus status_ = NOT_FOUND; + std::set> + call_handlers_; + }; + + void RegisterCallHandler( + const grpc::string& service_name, + std::shared_ptr handler); + + void UnregisterCallHandler( + const grpc::string& service_name, + std::shared_ptr handler); + mutable std::mutex mu_; - std::map services_map_; + std::map services_map_; // Guarded by mu_. std::unique_ptr impl_; }; diff --git a/src/cpp/server/health/health.pb.c b/src/cpp/server/health/health.pb.c index 09bd98a3d97..5c214c7160f 100644 --- a/src/cpp/server/health/health.pb.c +++ b/src/cpp/server/health/health.pb.c @@ -2,7 +2,6 @@ /* Generated by nanopb-0.3.7-dev */ #include "src/cpp/server/health/health.pb.h" - /* @@protoc_insertion_point(includes) */ #if PB_PROTO_HEADER_VERSION != 30 #error Regenerate this file with the current version of nanopb generator. diff --git a/src/cpp/server/health/health.pb.h b/src/cpp/server/health/health.pb.h index 29e1f3bacb1..9d54ccd6182 100644 --- a/src/cpp/server/health/health.pb.h +++ b/src/cpp/server/health/health.pb.h @@ -17,11 +17,12 @@ extern "C" { typedef enum _grpc_health_v1_HealthCheckResponse_ServingStatus { grpc_health_v1_HealthCheckResponse_ServingStatus_UNKNOWN = 0, grpc_health_v1_HealthCheckResponse_ServingStatus_SERVING = 1, - grpc_health_v1_HealthCheckResponse_ServingStatus_NOT_SERVING = 2 + grpc_health_v1_HealthCheckResponse_ServingStatus_NOT_SERVING = 2, + grpc_health_v1_HealthCheckResponse_ServingStatus_SERVICE_UNKNOWN = 3 } grpc_health_v1_HealthCheckResponse_ServingStatus; #define _grpc_health_v1_HealthCheckResponse_ServingStatus_MIN grpc_health_v1_HealthCheckResponse_ServingStatus_UNKNOWN -#define _grpc_health_v1_HealthCheckResponse_ServingStatus_MAX grpc_health_v1_HealthCheckResponse_ServingStatus_NOT_SERVING -#define _grpc_health_v1_HealthCheckResponse_ServingStatus_ARRAYSIZE ((grpc_health_v1_HealthCheckResponse_ServingStatus)(grpc_health_v1_HealthCheckResponse_ServingStatus_NOT_SERVING+1)) +#define _grpc_health_v1_HealthCheckResponse_ServingStatus_MAX grpc_health_v1_HealthCheckResponse_ServingStatus_SERVICE_UNKNOWN +#define _grpc_health_v1_HealthCheckResponse_ServingStatus_ARRAYSIZE ((grpc_health_v1_HealthCheckResponse_ServingStatus)(grpc_health_v1_HealthCheckResponse_ServingStatus_SERVICE_UNKNOWN+1)) /* Struct definitions */ typedef struct _grpc_health_v1_HealthCheckRequest { diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 36f7eb81f96..bf612d2d246 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -559,16 +559,20 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { // Only create default health check service when user did not provide an // explicit one. + ServerCompletionQueue* health_check_cq = nullptr; + DefaultHealthCheckService::HealthCheckServiceImpl* + default_health_check_service_impl = nullptr; if (health_check_service_ == nullptr && !health_check_service_disabled_ && DefaultHealthCheckServiceEnabled()) { - if (sync_server_cqs_ == nullptr || sync_server_cqs_->empty()) { - gpr_log(GPR_INFO, - "Default health check service disabled at async-only server."); - } else { - auto* default_hc_service = new DefaultHealthCheckService; - health_check_service_.reset(default_hc_service); - RegisterService(nullptr, default_hc_service->GetHealthCheckService()); - } + auto* default_hc_service = new DefaultHealthCheckService; + health_check_service_.reset(default_hc_service); + health_check_cq = new ServerCompletionQueue(GRPC_CQ_DEFAULT_POLLING); + grpc_server_register_completion_queue(server_, health_check_cq->cq(), + nullptr); + default_health_check_service_impl = + default_hc_service->GetHealthCheckService( + std::unique_ptr(health_check_cq)); + RegisterService(nullptr, default_health_check_service_impl); } grpc_server_start(server_); @@ -583,6 +587,9 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { new UnimplementedAsyncRequest(this, cqs[i]); } } + if (health_check_cq != nullptr) { + new UnimplementedAsyncRequest(this, health_check_cq); + } } // If this server has any support for synchronous methods (has any sync @@ -595,6 +602,10 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) { (*it)->Start(); } + + if (default_health_check_service_impl != nullptr) { + default_health_check_service_impl->StartServingThread(); + } } void Server::ShutdownInternal(gpr_timespec deadline) { diff --git a/src/proto/grpc/health/v1/health.proto b/src/proto/grpc/health/v1/health.proto index 4b4677b8a4d..38843ff1e73 100644 --- a/src/proto/grpc/health/v1/health.proto +++ b/src/proto/grpc/health/v1/health.proto @@ -34,10 +34,30 @@ message HealthCheckResponse { UNKNOWN = 0; SERVING = 1; NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; // Used only by the Watch method. } ServingStatus status = 1; } service Health { + // If the requested service is unknown, the call will fail with status + // NOT_FOUND. rpc Check(HealthCheckRequest) returns (HealthCheckResponse); + + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse); } diff --git a/test/cpp/end2end/health_service_end2end_test.cc b/test/cpp/end2end/health_service_end2end_test.cc index 1c48b9d151a..fca65dfc13b 100644 --- a/test/cpp/end2end/health_service_end2end_test.cc +++ b/test/cpp/end2end/health_service_end2end_test.cc @@ -64,6 +64,29 @@ class HealthCheckServiceImpl : public ::grpc::health::v1::Health::Service { return Status::OK; } + Status Watch(ServerContext* context, const HealthCheckRequest* request, + ::grpc::ServerWriter* writer) override { + auto last_state = HealthCheckResponse::UNKNOWN; + while (!context->IsCancelled()) { + { + std::lock_guard lock(mu_); + HealthCheckResponse response; + auto iter = status_map_.find(request->service()); + if (iter == status_map_.end()) { + response.set_status(response.SERVICE_UNKNOWN); + } else { + response.set_status(iter->second); + } + if (response.status() != last_state) { + writer->Write(response, ::grpc::WriteOptions()); + } + } + gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis(1000, GPR_TIMESPAN))); + } + return Status::OK; + } + void SetStatus(const grpc::string& service_name, HealthCheckResponse::ServingStatus status) { std::lock_guard lock(mu_); @@ -106,14 +129,6 @@ class CustomHealthCheckService : public HealthCheckServiceInterface { HealthCheckServiceImpl* impl_; // not owned }; -void LoopCompletionQueue(ServerCompletionQueue* cq) { - void* tag; - bool ok; - while (cq->Next(&tag, &ok)) { - abort(); // Nothing should come out of the cq. - } -} - class HealthServiceEnd2endTest : public ::testing::Test { protected: HealthServiceEnd2endTest() {} @@ -218,6 +233,33 @@ class HealthServiceEnd2endTest : public ::testing::Test { Status(StatusCode::NOT_FOUND, "")); } + void VerifyHealthCheckServiceStreaming() { + const grpc::string kServiceName("service_name"); + HealthCheckServiceInterface* service = server_->GetHealthCheckService(); + // Start Watch for service. + ClientContext context; + HealthCheckRequest request; + request.set_service(kServiceName); + std::unique_ptr<::grpc::ClientReaderInterface> reader = + hc_stub_->Watch(&context, request); + // Initial response will be SERVICE_UNKNOWN. + HealthCheckResponse response; + EXPECT_TRUE(reader->Read(&response)); + EXPECT_EQ(response.SERVICE_UNKNOWN, response.status()); + response.Clear(); + // Now set service to NOT_SERVING and make sure we get an update. + service->SetServingStatus(kServiceName, false); + EXPECT_TRUE(reader->Read(&response)); + EXPECT_EQ(response.NOT_SERVING, response.status()); + response.Clear(); + // Now set service to SERVING and make sure we get another update. + service->SetServingStatus(kServiceName, true); + EXPECT_TRUE(reader->Read(&response)); + EXPECT_EQ(response.SERVING, response.status()); + // Finish call. + context.TryCancel(); + } + TestServiceImpl echo_test_service_; HealthCheckServiceImpl health_check_service_impl_; std::unique_ptr hc_stub_; @@ -245,6 +287,7 @@ TEST_F(HealthServiceEnd2endTest, DefaultHealthService) { EXPECT_TRUE(DefaultHealthCheckServiceEnabled()); SetUpServer(true, false, false, nullptr); VerifyHealthCheckService(); + VerifyHealthCheckServiceStreaming(); // The default service has a size limit of the service name. const grpc::string kTooLongServiceName(201, 'x'); @@ -252,22 +295,6 @@ TEST_F(HealthServiceEnd2endTest, DefaultHealthService) { Status(StatusCode::INVALID_ARGUMENT, "")); } -// The server has no sync service. -TEST_F(HealthServiceEnd2endTest, DefaultHealthServiceAsyncOnly) { - EnableDefaultHealthCheckService(true); - EXPECT_TRUE(DefaultHealthCheckServiceEnabled()); - SetUpServer(false, true, false, nullptr); - cq_thread_ = std::thread(LoopCompletionQueue, cq_.get()); - - HealthCheckServiceInterface* default_service = - server_->GetHealthCheckService(); - EXPECT_TRUE(default_service == nullptr); - - ResetStubs(); - - SendHealthCheckRpc("", Status(StatusCode::UNIMPLEMENTED, "")); -} - // Provide an empty service to disable the default service. TEST_F(HealthServiceEnd2endTest, ExplicitlyDisableViaOverride) { EnableDefaultHealthCheckService(true); @@ -296,6 +323,7 @@ TEST_F(HealthServiceEnd2endTest, ExplicitlyOverride) { ResetStubs(); VerifyHealthCheckService(); + VerifyHealthCheckServiceStreaming(); } } // namespace diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 6b98619c320..1c2ef9b7686 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -16,6 +16,7 @@ set -ex readonly NANOPB_ALTS_TMP_OUTPUT="$(mktemp -d)" +readonly NANOPB_HEALTH_TMP_OUTPUT="$(mktemp -d)" readonly NANOPB_TMP_OUTPUT="$(mktemp -d)" readonly PROTOBUF_INSTALL_PREFIX="$(mktemp -d)" @@ -67,6 +68,23 @@ if ! diff -r "$NANOPB_TMP_OUTPUT" src/core/ext/filters/client_channel/lb_policy/ exit 2 fi +# +# checks for health.proto +# +readonly HEALTH_GRPC_OUTPUT_PATH='src/cpp/server/health' +# nanopb-compile the proto to a temp location +./tools/codegen/core/gen_nano_proto.sh \ + src/proto/grpc/health/v1/health.proto \ + "$NANOPB_HEALTH_TMP_OUTPUT" \ + "$HEALTH_GRPC_OUTPUT_PATH" +# compare outputs to checked compiled code +for NANOPB_OUTPUT_FILE in $NANOPB_HEALTH_TMP_OUTPUT/*.pb.*; do + if ! diff "$NANOPB_OUTPUT_FILE" "src/cpp/server/health/$(basename $NANOPB_OUTPUT_FILE)"; then + echo "Outputs differ: $NANOPB_HEALTH_TMP_OUTPUT vs $HEALTH_GRPC_OUTPUT_PATH" + exit 2 + fi +done + # # Checks for handshaker.proto and transport_security_common.proto # From 4d9ad11653f76787c9af76c62ae18f7c7ea53c3d Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 25 Sep 2018 11:44:28 -0700 Subject: [PATCH 13/53] Simplify call state logic, create non-polling CQ, and some cleanup. --- .../health/default_health_check_service.cc | 153 +++++++----------- .../health/default_health_check_service.h | 26 ++- src/cpp/server/server_cc.cc | 7 +- 3 files changed, 73 insertions(+), 113 deletions(-) diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index fc3db1bba73..e86c05c4b0b 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -132,13 +132,11 @@ DefaultHealthCheckService::HealthCheckServiceImpl::HealthCheckServiceImpl( std::unique_ptr cq) : database_(database), cq_(std::move(cq)) { // Add Check() method. - check_method_ = new internal::RpcServiceMethod( - kHealthCheckMethodName, internal::RpcMethod::NORMAL_RPC, nullptr); - AddMethod(check_method_); + AddMethod(new internal::RpcServiceMethod( + kHealthCheckMethodName, internal::RpcMethod::NORMAL_RPC, nullptr)); // Add Watch() method. - watch_method_ = new internal::RpcServiceMethod( - kHealthWatchMethodName, internal::RpcMethod::SERVER_STREAMING, nullptr); - AddMethod(watch_method_); + AddMethod(new internal::RpcServiceMethod( + kHealthWatchMethodName, internal::RpcMethod::SERVER_STREAMING, nullptr)); // Create serving thread. thread_ = std::unique_ptr<::grpc_core::Thread>( new ::grpc_core::Thread("grpc_health_check_service", Serve, this)); @@ -161,10 +159,6 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::StartServingThread() { void DefaultHealthCheckService::HealthCheckServiceImpl::Serve(void* arg) { HealthCheckServiceImpl* service = reinterpret_cast(arg); - // TODO(juanlishen): This is a workaround to wait for the cq to be ready. - // Need to figure out why cq is not ready after service starts. - gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_seconds(1, GPR_TIMESPAN))); CheckCallHandler::CreateAndStart(service->cq_.get(), service->database_, service); WatchCallHandler::CreateAndStart(service->cq_.get(), service->database_, @@ -289,13 +283,13 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: grpc::Status status = Status::OK; ByteBuffer response; if (!service_->DecodeRequest(request_, &service_name)) { - status = Status(StatusCode::INVALID_ARGUMENT, ""); + status = Status(StatusCode::INVALID_ARGUMENT, "could not parse request"); } else { ServingStatus serving_status = database_->GetServingStatus(service_name); if (serving_status == NOT_FOUND) { status = Status(StatusCode::NOT_FOUND, "service name unknown"); } else if (!service_->EncodeResponse(serving_status, &response)) { - status = Status(StatusCode::INTERNAL, ""); + status = Status(StatusCode::INTERNAL, "could not encode response"); } } // Send response. @@ -361,23 +355,18 @@ DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: : cq_(cq), database_(database), service_(service), - stream_(&ctx_), - call_state_(WAITING_FOR_CALL) {} + stream_(&ctx_) {} void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: OnCallReceived(std::shared_ptr self, bool ok) { - if (ok) { - call_state_ = CALL_RECEIVED; - } else { + if (!ok) { + // Server shutting down. + // // AsyncNotifyWhenDone() needs to be called before the call starts, but the // tag will not pop out if the call never starts ( // https://github.com/grpc/grpc/issues/10136). So we need to manually // release the ownership of the handler in this case. GPR_ASSERT(on_done_notified_.ReleaseHandler() != nullptr); - } - if (!ok || shutdown_) { - // The value of ok being false means that the server is shutting down. - Shutdown(std::move(self), "OnCallReceived"); return; } // Spawn a new handler instance to serve the next new client. Every handler @@ -385,25 +374,20 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: CreateAndStart(cq_, database_, service_); // Parse request. if (!service_->DecodeRequest(request_, &service_name_)) { - on_finish_done_ = - CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, - std::placeholders::_1, std::placeholders::_2), - std::move(self)); - stream_.Finish(Status(StatusCode::INVALID_ARGUMENT, ""), &on_finish_done_); - call_state_ = FINISH_CALLED; + SendFinish(std::move(self), + Status(StatusCode::INVALID_ARGUMENT, "could not parse request")); return; } // Register the call for updates to the service. gpr_log(GPR_DEBUG, - "[HCS %p] Health check watch started for service \"%s\" " - "(handler: %p)", + "[HCS %p] Health watch started for service \"%s\" (handler: %p)", service_, service_name_.c_str(), this); database_->RegisterCallHandler(service_name_, std::move(self)); } void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: SendHealth(std::shared_ptr self, ServingStatus status) { - std::unique_lock lock(mu_); + std::unique_lock lock(send_mu_); // If there's already a send in flight, cache the new status, and // we'll start a new send for it when the one in flight completes. if (send_in_flight_) { @@ -416,22 +400,19 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: SendHealthLocked(std::shared_ptr self, ServingStatus status) { + send_in_flight_ = true; + // Construct response. + ByteBuffer response; + bool success = service_->EncodeResponse(status, &response); + // Grab shutdown lock and send response. std::unique_lock cq_lock(service_->cq_shutdown_mu_); if (service_->shutdown_) { - cq_lock.release()->unlock(); - Shutdown(std::move(self), "SendHealthLocked"); + SendFinishLocked(std::move(self), Status::CANCELLED); return; } - send_in_flight_ = true; - call_state_ = SEND_MESSAGE_PENDING; - // Construct response. - ByteBuffer response; - if (!service_->EncodeResponse(status, &response)) { - on_finish_done_ = - CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, - std::placeholders::_1, std::placeholders::_2), - std::move(self)); - stream_.Finish(Status(StatusCode::INTERNAL, ""), &on_finish_done_); + if (!success) { + SendFinishLocked(std::move(self), + Status(StatusCode::INTERNAL, "could not encode response")); return; } next_ = CallableTag(std::bind(&WatchCallHandler::OnSendHealthDone, this, @@ -442,76 +423,60 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: OnSendHealthDone(std::shared_ptr self, bool ok) { - if (!ok || shutdown_) { - Shutdown(std::move(self), "OnSendHealthDone"); + if (!ok) { + SendFinish(std::move(self), Status::CANCELLED); return; } - call_state_ = CALL_RECEIVED; - { - std::unique_lock lock(mu_); - send_in_flight_ = false; - // If we got a new status since we started the last send, start a - // new send for it. - if (pending_status_ != NOT_FOUND) { - auto status = pending_status_; - pending_status_ = NOT_FOUND; - SendHealthLocked(std::move(self), status); - } + std::unique_lock lock(send_mu_); + send_in_flight_ = false; + // If we got a new status since we started the last send, start a + // new send for it. + if (pending_status_ != NOT_FOUND) { + auto status = pending_status_; + pending_status_ = NOT_FOUND; + SendHealthLocked(std::move(self), status); } } void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: - OnDoneNotified(std::shared_ptr self, bool ok) { - GPR_ASSERT(ok); - done_notified_ = true; - if (ctx_.IsCancelled()) { - is_cancelled_ = true; - } - gpr_log(GPR_DEBUG, - "[HCS %p] Healt check call is notified done (handler: %p, " - "is_cancelled: %d).", - service_, this, static_cast(is_cancelled_)); - Shutdown(std::move(self), "OnDoneNotified"); + SendFinish(std::shared_ptr self, const Status& status) { + if (finish_called_) return; + std::unique_lock cq_lock(service_->cq_shutdown_mu_); + if (!service_->shutdown_) return; + SendFinishLocked(std::move(self), status); } -// TODO(roth): This method currently assumes that there will be only one -// thread polling the cq and invoking the corresponding callbacks. If -// that changes, we will need to add synchronization here. void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: - Shutdown(std::shared_ptr self, const char* reason) { - if (!shutdown_) { - gpr_log(GPR_DEBUG, - "[HCS %p] Shutting down the handler (service_name: \"%s\", " - "handler: %p, reason: %s).", - service_, service_name_.c_str(), this, reason); - shutdown_ = true; - } - // OnCallReceived() may be called after OnDoneNotified(), so we need to - // try to Finish() every time we are in Shutdown(). - if (call_state_ >= CALL_RECEIVED && call_state_ < FINISH_CALLED) { - std::unique_lock lock(service_->cq_shutdown_mu_); - if (!service_->shutdown_) { - on_finish_done_ = - CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, - std::placeholders::_1, std::placeholders::_2), - std::move(self)); - // TODO(juanlishen): Maybe add a message proto for the client to - // explicitly cancel the stream so that we can return OK status in such - // cases. - stream_.Finish(Status::CANCELLED, &on_finish_done_); - call_state_ = FINISH_CALLED; - } - } + SendFinishLocked(std::shared_ptr self, const Status& status) { + on_finish_done_ = + CallableTag(std::bind(&WatchCallHandler::OnFinishDone, this, + std::placeholders::_1, std::placeholders::_2), + std::move(self)); + stream_.Finish(status, &on_finish_done_); + finish_called_ = true; } void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: OnFinishDone(std::shared_ptr self, bool ok) { if (ok) { gpr_log(GPR_DEBUG, - "[HCS %p] Health check call finished (service_name: \"%s\", " + "[HCS %p] Health watch call finished (service_name: \"%s\", " "handler: %p).", service_, service_name_.c_str(), this); } } +// TODO(roth): This method currently assumes that there will be only one +// thread polling the cq and invoking the corresponding callbacks. If +// that changes, we will need to add synchronization here. +void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: + OnDoneNotified(std::shared_ptr self, bool ok) { + GPR_ASSERT(ok); + gpr_log(GPR_DEBUG, + "[HCS %p] Healt watch call is notified done (handler: %p, " + "is_cancelled: %d).", + service_, this, static_cast(ctx_.IsCancelled())); + SendFinish(std::move(self), Status::CANCELLED); +} + } // namespace grpc diff --git a/src/cpp/server/health/default_health_check_service.h b/src/cpp/server/health/default_health_check_service.h index edad5949362..3bab76b6b03 100644 --- a/src/cpp/server/health/default_health_check_service.h +++ b/src/cpp/server/health/default_health_check_service.h @@ -168,21 +168,25 @@ class DefaultHealthCheckService final : public HealthCheckServiceInterface { // Spawns a new handler so that we can keep servicing future calls. void OnCallReceived(std::shared_ptr self, bool ok); - // Requires holding mu_. + // Requires holding send_mu_. void SendHealthLocked(std::shared_ptr self, ServingStatus status); // When sending a health result finishes. void OnSendHealthDone(std::shared_ptr self, bool ok); + void SendFinish(std::shared_ptr self, const Status& status); + + // Requires holding service_->cq_shutdown_mu_. + void SendFinishLocked(std::shared_ptr self, + const Status& status); + // Called when Finish() is done. void OnFinishDone(std::shared_ptr self, bool ok); // Called when AsyncNotifyWhenDone() notifies us. void OnDoneNotified(std::shared_ptr self, bool ok); - void Shutdown(std::shared_ptr self, const char* reason); - // The members passed down from HealthCheckServiceImpl. ServerCompletionQueue* cq_; DefaultHealthCheckService* database_; @@ -193,21 +197,11 @@ class DefaultHealthCheckService final : public HealthCheckServiceInterface { GenericServerAsyncWriter stream_; ServerContext ctx_; - std::mutex mu_; + std::mutex send_mu_; bool send_in_flight_ = false; // Guarded by mu_. ServingStatus pending_status_ = NOT_FOUND; // Guarded by mu_. - // The state of the RPC progress. - enum CallState { - WAITING_FOR_CALL, - CALL_RECEIVED, - SEND_MESSAGE_PENDING, - FINISH_CALLED - } call_state_; - - bool shutdown_ = false; - bool done_notified_ = false; - bool is_cancelled_ = false; + bool finish_called_ = false; CallableTag next_; CallableTag on_done_notified_; CallableTag on_finish_done_; @@ -229,8 +223,6 @@ class DefaultHealthCheckService final : public HealthCheckServiceInterface { DefaultHealthCheckService* database_; std::unique_ptr cq_; - internal::RpcServiceMethod* check_method_; - internal::RpcServiceMethod* watch_method_; // To synchronize the operations related to shutdown state of cq_, so that // we don't enqueue new tags into cq_ after it is already shut down. diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index bf612d2d246..f1f971bf74d 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -380,7 +380,6 @@ class Server::SyncRequestThreadManager : public ThreadManager { int cq_timeout_msec_; std::vector> sync_requests_; std::unique_ptr unknown_method_; - std::unique_ptr health_check_; std::shared_ptr global_callbacks_; }; @@ -566,7 +565,11 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { DefaultHealthCheckServiceEnabled()) { auto* default_hc_service = new DefaultHealthCheckService; health_check_service_.reset(default_hc_service); - health_check_cq = new ServerCompletionQueue(GRPC_CQ_DEFAULT_POLLING); + // We create a non-polling CQ to avoid impacting application + // performance. This ensures that we don't introduce thread hops + // for application requests that wind up on this CQ, which is polled + // in its own thread. + health_check_cq = new ServerCompletionQueue(GRPC_CQ_NON_POLLING); grpc_server_register_completion_queue(server_, health_check_cq->cq(), nullptr); default_health_check_service_impl = From a844323c7e96d7a525b963769c2a8e7db5ea4b61 Mon Sep 17 00:00:00 2001 From: kkm Date: Sun, 7 Oct 2018 00:39:02 -0700 Subject: [PATCH 14/53] Rename test classes *Test; UWYU in Tools.Test project --- .../Grpc.Tools.Tests/CSharpGeneratorTests.cs | 7 +------ src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs | 7 ++----- src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs | 6 ++---- src/csharp/Grpc.Tools.Tests/GeneratorTests.cs | 5 ++--- src/csharp/Grpc.Tools.Tests/NUnitMain.cs | 2 +- .../Grpc.Tools.Tests/ProtoCompileBasicTests.cs | 6 +++--- .../ProtoCompileCommandLineGeneratorTests.cs | 5 ++--- .../ProtoCompileCommandLinePrinterTests.cs | 5 ++--- ...askTest.cs => ProtoToolsPlatformTaskTests.cs} | 4 ++-- src/csharp/Grpc.Tools.Tests/Utils.cs | 3 --- src/csharp/tests.json | 16 ++++++++-------- 11 files changed, 25 insertions(+), 41 deletions(-) rename src/csharp/Grpc.Tools.Tests/{ProtoToolsPlatformTaskTest.cs => ProtoToolsPlatformTaskTests.cs} (97%) diff --git a/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs index dbffa65d8c8..654500d53d0 100644 --- a/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs +++ b/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs @@ -16,15 +16,10 @@ #endregion -using System.IO; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - - public class CSharpGeneratorTests : GeneratorTests { + public class CSharpGeneratorTest : GeneratorTest { GeneratorServices _generator; [SetUp] diff --git a/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs index e25a6498cd8..a3450fae176 100644 --- a/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs +++ b/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs @@ -17,13 +17,10 @@ #endregion using System.IO; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - public class CppGeneratorTests : GeneratorTests { + public class CppGeneratorTest : GeneratorTest { GeneratorServices _generator; [SetUp] @@ -79,5 +76,5 @@ namespace Grpc.Tools.Tests { Assert.AreEqual(2, poss.Length); Assert.That(Path.GetDirectoryName(poss[0]), Is.EqualTo("out")); } - } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs b/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs index 42d0ae5998b..ea34c899213 100644 --- a/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs +++ b/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs @@ -16,15 +16,13 @@ #endregion -using System; using System.IO; -using Grpc.Tools; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using NUnit.Framework; namespace Grpc.Tools.Tests { - public class DepFileUtilTests { + public class DepFileUtilTest { [Test] public void HashString64Hex_IsSane() { @@ -130,5 +128,5 @@ obj\Release x64\net45\/FooGrpc.cs: C:/usr/include/google/protobuf/wrappers.proto } catch { } } } - } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs index 93df9d6cab2..52fab1d8ca9 100644 --- a/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs +++ b/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs @@ -16,14 +16,13 @@ #endregion -using System.IO; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - public class GeneratorTests { + public class GeneratorTest { protected Mock _mockEngine; protected TaskLoggingHelper _log; @@ -47,5 +46,5 @@ namespace Grpc.Tools.Tests { Assert.IsNull(GeneratorServices.GetForLanguage(lang, _log)); _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Once); } - } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs index c4452c50d2c..5a8ae8cf873 100644 --- a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs +++ b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs @@ -19,7 +19,7 @@ using System.Reflection; using NUnitLite; -namespace Grps.Tools.Tests { +namespace Grpc.Tools.Tests { static class NUnitMain { public static int Main(string[] args) => #if NETCOREAPP1_0 diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs index 0660b3e33d4..cf9d210424c 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs @@ -16,13 +16,13 @@ #endregion -using System.Reflection; +using System.Reflection; // UWYU: Object.GetType() extension. using Microsoft.Build.Framework; using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - public class ProtoCompileBasicTests { + public class ProtoCompileBasicTest { // Mock task class that stops right before invoking protoc. public class ProtoCompileTestable : ProtoCompile { public string LastPathToTool { get; private set; } @@ -66,5 +66,5 @@ namespace Grpc.Tools.Tests { Assert.NotNull(pinfo); Assert.That(pinfo, Has.Attribute()); } - } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs index fcdff11da45..c5d43f08d99 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs @@ -16,13 +16,12 @@ #endregion -using System.Reflection; using Microsoft.Build.Framework; using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - internal class ProtoCompileCommandLineGeneratorTests : ProtoCompileBasicTests { + public class ProtoCompileCommandLineGeneratorTest : ProtoCompileBasicTest { [SetUp] public new void SetUp() { _task.Generator = "csharp"; @@ -158,5 +157,5 @@ namespace Grpc.Tools.Tests { Assert.That(_task.LastResponseFile, Does.Contain("--csharp_out=" + expect)); } - } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs index 91cfceaf568..a0406371dcc 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs @@ -16,13 +16,12 @@ #endregion -using System.Reflection; using Microsoft.Build.Framework; using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - internal class ProtoCompileCommandLinePrinterTests : ProtoCompileBasicTests { + public class ProtoCompileCommandLinePrinterTest : ProtoCompileBasicTest { [SetUp] public new void SetUp() { _task.Generator = "csharp"; @@ -44,5 +43,5 @@ namespace Grpc.Tools.Tests { bool result = _task.Execute(); Assert.IsTrue(result); } - } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTests.cs similarity index 97% rename from src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs rename to src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTests.cs index 3ba0bdfbf6b..235f8de5275 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTests.cs @@ -22,11 +22,11 @@ using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - // This test requires that environment variables be set to the exected + // This test requires that environment variables be set to the expected // output of the task in its external test harness: // PROTOTOOLS_TEST_CPU = { x64 | x86 } // PROTOTOOLS_TEST_OS = { linux | macosx | windows } - public class ProtoToolsPlatformTaskTests { + public class ProtoToolsPlatformTaskTest { static string s_expectOs; static string s_expectCpu; diff --git a/src/csharp/Grpc.Tools.Tests/Utils.cs b/src/csharp/Grpc.Tools.Tests/Utils.cs index 618e3354524..bb051a4873f 100644 --- a/src/csharp/Grpc.Tools.Tests/Utils.cs +++ b/src/csharp/Grpc.Tools.Tests/Utils.cs @@ -16,10 +16,7 @@ #endregion -using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; diff --git a/src/csharp/tests.json b/src/csharp/tests.json index 761308b33fb..760776f9e70 100644 --- a/src/csharp/tests.json +++ b/src/csharp/tests.json @@ -66,13 +66,13 @@ "Grpc.Reflection.Tests.SymbolRegistryTest" ], "Grpc.Tools.Tests": [ - "Grpc.Tools.Tests.CppGeneratorTests", - "Grpc.Tools.Tests.CSharpGeneratorTests", - "Grpc.Tools.Tests.DepFileUtilTests", - "Grpc.Tools.Tests.GeneratorTests", - "Grpc.Tools.Tests.ProtoCompileBasicTests", - "Grpc.Tools.Tests.ProtoCompileCommandLineGeneratorTests", - "Grpc.Tools.Tests.ProtoCompileCommandLinePrinterTests", - "Grpc.Tools.Tests.ProtoToolsPlatformTaskTests" + "Grpc.Tools.Tests.CppGeneratorTest", + "Grpc.Tools.Tests.CSharpGeneratorTest", + "Grpc.Tools.Tests.DepFileUtilTest", + "Grpc.Tools.Tests.GeneratorTest", + "Grpc.Tools.Tests.ProtoCompileBasicTest", + "Grpc.Tools.Tests.ProtoCompileCommandLineGeneratorTest", + "Grpc.Tools.Tests.ProtoCompileCommandLinePrinterTest", + "Grpc.Tools.Tests.ProtoToolsPlatformTaskTest" ] } From 716b5577fc7c744bb85730d134366f17f71ea00e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 Oct 2018 13:09:41 +0200 Subject: [PATCH 15/53] fixup: rename *Tests files to *Test --- .../{CSharpGeneratorTests.cs => CSharpGeneratorTest.cs} | 0 .../{CppGeneratorTests.cs => CppGeneratorTest.cs} | 0 .../Grpc.Tools.Tests/{DepFileUtilTests.cs => DepFileUtilTest.cs} | 0 .../Grpc.Tools.Tests/{GeneratorTests.cs => GeneratorTest.cs} | 0 .../{ProtoCompileBasicTests.cs => ProtoCompileBasicTest.cs} | 0 ...eGeneratorTests.cs => ProtoCompileCommandLineGeneratorTest.cs} | 0 ...dLinePrinterTests.cs => ProtoCompileCommandLinePrinterTest.cs} | 0 ...otoToolsPlatformTaskTests.cs => ProtoToolsPlatformTaskTest.cs} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename src/csharp/Grpc.Tools.Tests/{CSharpGeneratorTests.cs => CSharpGeneratorTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{CppGeneratorTests.cs => CppGeneratorTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{DepFileUtilTests.cs => DepFileUtilTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{GeneratorTests.cs => GeneratorTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{ProtoCompileBasicTests.cs => ProtoCompileBasicTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{ProtoCompileCommandLineGeneratorTests.cs => ProtoCompileCommandLineGeneratorTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{ProtoCompileCommandLinePrinterTests.cs => ProtoCompileCommandLinePrinterTest.cs} (100%) rename src/csharp/Grpc.Tools.Tests/{ProtoToolsPlatformTaskTests.cs => ProtoToolsPlatformTaskTest.cs} (100%) diff --git a/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/CSharpGeneratorTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/CSharpGeneratorTests.cs rename to src/csharp/Grpc.Tools.Tests/CSharpGeneratorTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/CppGeneratorTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/CppGeneratorTests.cs rename to src/csharp/Grpc.Tools.Tests/CppGeneratorTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs b/src/csharp/Grpc.Tools.Tests/DepFileUtilTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/DepFileUtilTests.cs rename to src/csharp/Grpc.Tools.Tests/DepFileUtilTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/GeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/GeneratorTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/GeneratorTests.cs rename to src/csharp/Grpc.Tools.Tests/GeneratorTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTests.cs rename to src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTests.cs rename to src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTests.cs rename to src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTest.cs diff --git a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTests.cs b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs similarity index 100% rename from src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTests.cs rename to src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs From ccacf24fb077e9de1f1bfb3af9ad33fb5de71870 Mon Sep 17 00:00:00 2001 From: kkm Date: Thu, 11 Oct 2018 13:12:01 -0700 Subject: [PATCH 16/53] Fix dotnet/Mono build and testing under Linux --- .../Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 49 +++++++++++++++---- src/csharp/Grpc.Tools.Tests/NUnitMain.cs | 2 +- .../ProtoCompileCommandLineGeneratorTest.cs | 3 ++ src/csharp/Grpc.Tools/Common.cs | 2 +- src/csharp/Grpc.Tools/Grpc.Tools.csproj | 47 ++++++++---------- 5 files changed, 66 insertions(+), 37 deletions(-) diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index ab14f32afd4..70c0474047e 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -3,7 +3,7 @@ - net45;netcoreapp1.0 + net45;netcoreapp1.0;netcoreapp2.1 Exe @@ -14,8 +14,8 @@ and may not compile Grpc.Core/Version.cs, as that file references constants in Grpc.Core.dll. TODO(kkm): Refactor imports. --> - - + + /usr/lib/mono/4.5-api /usr/local/lib/mono/4.5-api /Library/Frameworks/Mono.framework/Versions/Current/lib/mono/4.5-api @@ -27,16 +27,47 @@ - - - - - - + + + + + + + + + + + + + + {HintPathFromItem};{TargetFrameworkDirectory};{RawFileName} + + <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' == 'Core' " + >$(FrameworkPathOverride)/../msbuild/$(MSBuildToolsVersion)/bin + + <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' != 'Core' " + >$(MSBuildToolsPath) + + diff --git a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs index 5a8ae8cf873..5784cdeac25 100644 --- a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs +++ b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs @@ -22,7 +22,7 @@ using NUnitLite; namespace Grpc.Tools.Tests { static class NUnitMain { public static int Main(string[] args) => -#if NETCOREAPP1_0 +#if NETCOREAPP1_0 || NETCOREAPP1_1 new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args); #else new AutoRun().Execute(args); diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs index c5d43f08d99..06376f8ef4e 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs @@ -16,6 +16,7 @@ #endregion +using System.IO; using Microsoft.Build.Framework; using Moq; using NUnit.Framework; @@ -152,6 +153,8 @@ namespace Grpc.Tools.Tests { [TestCase("C:\\" , "C:\\")] [TestCase("C:\\\\" , "C:\\")] public void DirectorySlashTrimmingCases(string given, string expect) { + if (Path.DirectorySeparatorChar == '/') + expect = expect.Replace('\\', '/'); _task.OutputDir = given; ExecuteExpectSuccess(); Assert.That(_task.LastResponseFile, diff --git a/src/csharp/Grpc.Tools/Common.cs b/src/csharp/Grpc.Tools/Common.cs index 1ebd386bd1c..9f8600bad0c 100644 --- a/src/csharp/Grpc.Tools/Common.cs +++ b/src/csharp/Grpc.Tools/Common.cs @@ -50,7 +50,7 @@ namespace Grpc.Tools { public static bool IsWindows => Os == OsKind.Windows; static Platform() { -#if NETSTANDARD +#if NETCORE Os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? OsKind.Windows : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? OsKind.Linux : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? OsKind.MacOsX diff --git a/src/csharp/Grpc.Tools/Grpc.Tools.csproj b/src/csharp/Grpc.Tools/Grpc.Tools.csproj index 8edfb848d72..d93947b8ffe 100644 --- a/src/csharp/Grpc.Tools/Grpc.Tools.csproj +++ b/src/csharp/Grpc.Tools/Grpc.Tools.csproj @@ -14,8 +14,8 @@ and may not compile Grpc.Core/Version.cs, as that file references constants in Grpc.Core.dll. TODO(kkm): Refactor imports. --> - - + + /usr/lib/mono/4.5-api /usr/local/lib/mono/4.5-api /Library/Frameworks/Mono.framework/Versions/Current/lib/mono/4.5-api @@ -27,20 +27,16 @@ ../../../third_party/protobuf/src/google/protobuf/ - - + + ../protoc_plugins/protoc_ - + ../protoc_plugins/ - - <_NetStandard>False - <_NetStandard Condition=" $(TargetFramework.StartsWith('netstandard')) or $(TargetFramework.StartsWith('netcore')) ">True - - - $(DefineConstants);NETSTANDARD + + $(DefineConstants);NETCORE @@ -71,29 +67,28 @@ Linux and MacOS. Managed runtime is supplied separately in the Grpc.Core package <_Asset PackagePath="build/native/include/google/protobuf/" Include="@(_ProtoAssetName->'$(Assets_ProtoInclude)%(Identity).proto')" /> - <_Asset PackagePath="build/native/bin/windows/protoc.exe" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> - <_Asset PackagePath="build/native/bin/linux_x86/protoc" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> - <_Asset PackagePath="build/native/bin/linux_x64/protoc" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> - <_Asset PackagePath="build/native/bin/macosx_x86/protoc" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> - <_Asset PackagePath="build/native/bin/macosx_x64/protoc" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> + <_Asset PackagePath="build/native/bin/windows/" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> + <_Asset PackagePath="build/native/bin/linux_x86/" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> + <_Asset PackagePath="build/native/bin/linux_x64/" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> + <_Asset PackagePath="build/native/bin/macosx_x86/" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> + <_Asset PackagePath="build/native/bin/macosx_x64/" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> - <_Asset PackagePath="build/native/bin/windows/grpc_csharp_plugin.exe" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> - <_Asset PackagePath="build/native/bin/linux_x86/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> - <_Asset PackagePath="build/native/bin/linux_x64/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> - <_Asset PackagePath="build/native/bin/macosx_x86/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> - <_Asset PackagePath="build/native/bin/macosx_x64/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/windows/" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> + <_Asset PackagePath="build/native/bin/linux_x86/" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/linux_x64/" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/macosx_x86/" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="build/native/bin/macosx_x64/" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> - - - + + - - + + From 9158493c41f4d47f5409e40760afe6babf17bb2e Mon Sep 17 00:00:00 2001 From: kkm Date: Sun, 14 Oct 2018 04:17:08 -0700 Subject: [PATCH 17/53] Improve ProtoToolsPlatformTask tests --- .../Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 13 +- .../ProtoToolsPlatformTaskTest.cs | 111 ++++++++++++++---- 2 files changed, 99 insertions(+), 25 deletions(-) diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index 70c0474047e..cb065d3c368 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -28,8 +28,13 @@ + + + $(DefineConstants);NETCORE + + @@ -62,12 +67,12 @@ and does not define any types at all). Exclude them from assembly resolution. See https://github.com/Microsoft/msbuild/blob/50639058f/documentation/wiki/ResolveAssemblyReference.md --> {HintPathFromItem};{TargetFrameworkDirectory};{RawFileName} - - <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' == 'Core' " - >$(FrameworkPathOverride)/../msbuild/$(MSBuildToolsVersion)/bin - + <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' != 'Core' " >$(MSBuildToolsPath) + + <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' == 'Core' " + >$(FrameworkPathOverride)/../msbuild/$(MSBuildToolsVersion)/bin diff --git a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs index 235f8de5275..2380ae8a37e 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs @@ -16,38 +16,107 @@ #endregion -using System; +using System.Runtime.InteropServices; // For NETCORE tests. using Microsoft.Build.Framework; using Moq; using NUnit.Framework; namespace Grpc.Tools.Tests { - // This test requires that environment variables be set to the expected - // output of the task in its external test harness: - // PROTOTOOLS_TEST_CPU = { x64 | x86 } - // PROTOTOOLS_TEST_OS = { linux | macosx | windows } public class ProtoToolsPlatformTaskTest { - static string s_expectOs; - static string s_expectCpu; + ProtoToolsPlatform _task; + int _cpuMatched, _osMatched; [OneTimeSetUp] - public static void Init() { - s_expectCpu = Environment.GetEnvironmentVariable("PROTOTOOLS_TEST_CPU"); - s_expectOs = Environment.GetEnvironmentVariable("PROTOTOOLS_TEST_OS"); - if (s_expectCpu == null || s_expectOs == null) - Assert.Inconclusive("This test requires PROTOTOOLS_TEST_CPU and " + - "PROTOTOOLS_TEST_OS set in the environment to match the OS it runs on."); - } - - [Test] - public void CpuAndOsMatchExpected() { + public void SetUp() { var mockEng = new Mock(); - var task = new ProtoToolsPlatform() { + _task = new ProtoToolsPlatform() { BuildEngine = mockEng.Object }; - task.Execute(); - Assert.AreEqual(s_expectCpu, task.Cpu); - Assert.AreEqual(s_expectOs, task.Os); + _task.Execute(); + } + + [OneTimeTearDown] + public void TearDown() { + Assert.AreEqual(1, _cpuMatched, "CPU type detection failed"); + Assert.AreEqual(1, _osMatched, "OS detection failed"); + } + +#if NETCORE + // PlatformAttribute not yet available in NUnit, coming soon: + // https://github.com/nunit/nunit/pull/3003. + // Use same test case names as under the full framework. + [Test] + public void CpuIsX86() { + if (RuntimeInformation.OSArchitecture == Architecture.X86) { + _cpuMatched++; + Assert.AreEqual("x86", _task.Cpu); + } + } + + [Test] + public void CpuIsX64() { + if (RuntimeInformation.OSArchitecture == Architecture.X64) { + _cpuMatched++; + Assert.AreEqual("x64", _task.Cpu); + } + } + + [Test] + public void OsIsWindows() { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + _osMatched++; + Assert.AreEqual("windows", _task.Os); + } + } + + [Test] + public void OsIsLinux() { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { + _osMatched++; + Assert.AreEqual("linux", _task.Os); + } + } + + [Test] + public void OsIsMacOsX() { + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { + _osMatched++; + Assert.AreEqual("macosx", _task.Os); + } + } + +#else // !NETCORE, i.e. full framework. + + [Test, Platform("32-Bit")] + public void CpuIsX86() { + _cpuMatched++; + Assert.AreEqual("x86", _task.Cpu); + } + + [Test, Platform("64-Bit")] + public void CpuIsX64() { + _cpuMatched++; + Assert.AreEqual("x64", _task.Cpu); + } + + [Test, Platform("Win")] + public void OsIsWindows() { + _osMatched++; + Assert.AreEqual("windows", _task.Os); + } + + [Test, Platform("Linux")] + public void OsIsLinux() { + _osMatched++; + Assert.AreEqual("linux", _task.Os); + } + + [Test, Platform("MacOSX")] + public void OsIsMacOsX() { + _osMatched++; + Assert.AreEqual("macosx", _task.Os); } + +#endif // NETCORE }; } From f626d4618d8ed15853e763e96a23e3635be0c339 Mon Sep 17 00:00:00 2001 From: kkm Date: Sun, 14 Oct 2018 06:59:49 -0700 Subject: [PATCH 18/53] Restore packaging of binary tools under tools/ Unfixes #13098, hopefully temporarily. --- src/csharp/Grpc.Tools/Grpc.Tools.csproj | 24 +++++++++++-------- .../build/_grpc/_Grpc.Tools.targets | 8 ++++--- .../_protobuf/Google.Protobuf.Tools.props | 7 +++--- .../_protobuf/Google.Protobuf.Tools.targets | 5 ++-- .../Grpc.Tools/build/native/Grpc.Tools.props | 8 ++++--- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/csharp/Grpc.Tools/Grpc.Tools.csproj b/src/csharp/Grpc.Tools/Grpc.Tools.csproj index d93947b8ffe..f2cefc9826b 100644 --- a/src/csharp/Grpc.Tools/Grpc.Tools.csproj +++ b/src/csharp/Grpc.Tools/Grpc.Tools.csproj @@ -67,18 +67,22 @@ Linux and MacOS. Managed runtime is supplied separately in the Grpc.Core package <_Asset PackagePath="build/native/include/google/protobuf/" Include="@(_ProtoAssetName->'$(Assets_ProtoInclude)%(Identity).proto')" /> - <_Asset PackagePath="build/native/bin/windows/" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> - <_Asset PackagePath="build/native/bin/linux_x86/" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> - <_Asset PackagePath="build/native/bin/linux_x64/" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> - <_Asset PackagePath="build/native/bin/macosx_x86/" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> - <_Asset PackagePath="build/native/bin/macosx_x64/" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> + + + <_Asset PackagePath="tools/windows_x86/" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> + <_Asset PackagePath="tools/windows_x64/" Include="$(Assets_ProtoCompiler)windows_x64/protoc.exe" /> + <_Asset PackagePath="tools/linux_x86/" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> + <_Asset PackagePath="tools/linux_x64/" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> + <_Asset PackagePath="tools/macosx_x86/" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> + <_Asset PackagePath="tools/macosx_x64/" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> - <_Asset PackagePath="build/native/bin/windows/" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> - <_Asset PackagePath="build/native/bin/linux_x86/" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> - <_Asset PackagePath="build/native/bin/linux_x64/" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> - <_Asset PackagePath="build/native/bin/macosx_x86/" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> - <_Asset PackagePath="build/native/bin/macosx_x64/" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/windows_x86/" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> + <_Asset PackagePath="tools/windows_x64/" Include="$(Assets_GrpcPlugins)protoc_windows_x64/grpc_csharp_plugin.exe" /> + <_Asset PackagePath="tools/linux_x86/" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/linux_x64/" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/macosx_x86/" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/macosx_x64/" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> diff --git a/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets b/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets index 0042bf2bfa9..5f76c03ce57 100644 --- a/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets +++ b/src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets @@ -14,17 +14,19 @@ - Both + Both + + $(Protobuf_PackagedToolsPath)bin\$(Protobuf_ToolsOs)\$(gRPC_PluginFileName).exe + >$(Protobuf_PackagedToolsPath)\$(Protobuf_ToolsOs)_x86\$(gRPC_PluginFileName).exe $(Protobuf_PackagedToolsPath)bin/$(Protobuf_ToolsOs)_$(Protobuf_ToolsCpu)/$(gRPC_PluginFileName) + >$(Protobuf_PackagedToolsPath)/$(Protobuf_ToolsOs)_$(Protobuf_ToolsCpu)/$(gRPC_PluginFileName) diff --git a/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props index 06ee9bcda8a..9f2d8bb4b5c 100644 --- a/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props +++ b/src/csharp/Grpc.Tools/build/_protobuf/Google.Protobuf.Tools.props @@ -6,9 +6,10 @@ 1 - - $( [System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)../native/) ) - $(Protobuf_PackagedToolsPath)include + + + $( [System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)../../tools) ) + $( [System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)../native/include) ) $(_Protobuf_ToolsOs) $(_Protobuf_ToolsCpu) + $(Protobuf_PackagedToolsPath)bin\$(Protobuf_ToolsOs)\protoc.exe + >$(Protobuf_PackagedToolsPath)\$(Protobuf_ToolsOs)_x86\protoc.exe $(Protobuf_PackagedToolsPath)bin/$(Protobuf_ToolsOs)_$(Protobuf_ToolsCpu)/protoc + >$(Protobuf_PackagedToolsPath)/$(Protobuf_ToolsOs)_$(Protobuf_ToolsCpu)/protoc - DepFileUtil.GetDepFilenameForProto("", fname).Substring(0, 16); - - string same1 = PickHash("dir1/dir2/foo.proto"); - string same2 = PickHash("dir1/dir2/proto.foo"); - string same3 = PickHash("dir1/dir2/proto"); - string same4 = PickHash("dir1/dir2/.proto"); - string unsame1 = PickHash("dir2/foo.proto"); - string unsame2 = PickHash("/dir2/foo.proto"); - - Assert.AreEqual(same1, same2); - Assert.AreEqual(same1, same3); - Assert.AreEqual(same1, same4); - Assert.AreNotEqual(same1, unsame1); - Assert.AreNotEqual(unsame1, unsame2); - } - - ////////////////////////////////////////////////////////////////////////// - // Full file reading tests - - // Generated by protoc on Windows. Slashes vary. - const string depFile1 = -@"C:\projects\foo\src\./foo.grpc.pb.cc \ +namespace Grpc.Tools.Tests +{ + public class DepFileUtilTest + { + + [Test] + public void HashString64Hex_IsSane() + { + string hashFoo1 = DepFileUtil.HashString64Hex("foo"); + string hashEmpty = DepFileUtil.HashString64Hex(""); + string hashFoo2 = DepFileUtil.HashString64Hex("foo"); + + StringAssert.IsMatch("^[a-f0-9]{16}$", hashFoo1); + Assert.AreEqual(hashFoo1, hashFoo2); + Assert.AreNotEqual(hashFoo1, hashEmpty); + } + + [Test] + public void GetDepFilenameForProto_IsSane() + { + StringAssert.IsMatch(@"^out[\\/][a-f0-9]{16}_foo.protodep$", + DepFileUtil.GetDepFilenameForProto("out", "foo.proto")); + StringAssert.IsMatch(@"^[a-f0-9]{16}_foo.protodep$", + DepFileUtil.GetDepFilenameForProto("", "foo.proto")); + } + + [Test] + public void GetDepFilenameForProto_HashesDir() + { + string PickHash(string fname) => + DepFileUtil.GetDepFilenameForProto("", fname).Substring(0, 16); + + string same1 = PickHash("dir1/dir2/foo.proto"); + string same2 = PickHash("dir1/dir2/proto.foo"); + string same3 = PickHash("dir1/dir2/proto"); + string same4 = PickHash("dir1/dir2/.proto"); + string unsame1 = PickHash("dir2/foo.proto"); + string unsame2 = PickHash("/dir2/foo.proto"); + + Assert.AreEqual(same1, same2); + Assert.AreEqual(same1, same3); + Assert.AreEqual(same1, same4); + Assert.AreNotEqual(same1, unsame1); + Assert.AreNotEqual(unsame1, unsame2); + } + + ////////////////////////////////////////////////////////////////////////// + // Full file reading tests + + // Generated by protoc on Windows. Slashes vary. + const string depFile1 = + @"C:\projects\foo\src\./foo.grpc.pb.cc \ C:\projects\foo\src\./foo.grpc.pb.h \ C:\projects\foo\src\./foo.pb.cc \ C:\projects\foo\src\./foo.pb.h: C:/usr/include/google/protobuf/wrappers.proto\ @@ -76,57 +81,66 @@ C:/usr/include/google/protobuf/source_context.proto\ C:/usr/include/google/protobuf/type.proto\ foo.proto"; - // This has a nasty output directory with a space. - const string depFile2 = -@"obj\Release x64\net45\/Foo.cs \ + // This has a nasty output directory with a space. + const string depFile2 = + @"obj\Release x64\net45\/Foo.cs \ obj\Release x64\net45\/FooGrpc.cs: C:/usr/include/google/protobuf/wrappers.proto\ C:/projects/foo/src//foo.proto"; - [Test] - public void ReadDependencyInput_FullFile1() { - string[] deps = ReadDependencyInputFromFileData(depFile1, "foo.proto"); - - Assert.NotNull(deps); - Assert.That(deps, Has.Length.InRange(4, 5)); // foo.proto may or may not be listed. - Assert.That(deps, Has.One.EndsWith("wrappers.proto")); - Assert.That(deps, Has.One.EndsWith("type.proto")); - Assert.That(deps, Has.None.StartWith(" ")); - } - - [Test] - public void ReadDependencyInput_FullFile2() { - string[] deps = ReadDependencyInputFromFileData(depFile2, "C:/projects/foo/src/foo.proto"); - - Assert.NotNull(deps); - Assert.That(deps, Has.Length.InRange(1, 2)); - Assert.That(deps, Has.One.EndsWith("wrappers.proto")); - Assert.That(deps, Has.None.StartWith(" ")); - } - - [Test] - public void ReadDependencyInput_FullFileUnparsable() { - string[] deps = ReadDependencyInputFromFileData("a:/foo.proto", "/foo.proto"); - Assert.NotNull(deps); - Assert.Zero(deps.Length); - } - - // NB in our tests files are put into the temp directory but all have - // different names. Avoid adding files with the same directory path and - // name, or add reasonable handling for it if required. Tests are run in - // parallel and will collide otherwise. - private string[] ReadDependencyInputFromFileData(string fileData, string protoName) { - string tempPath = Path.GetTempPath(); - string tempfile = DepFileUtil.GetDepFilenameForProto(tempPath, protoName); - try { - File.WriteAllText(tempfile, fileData); - var mockEng = new Moq.Mock(); - var log = new TaskLoggingHelper(mockEng.Object, "x"); - return DepFileUtil.ReadDependencyInputs(tempPath, protoName, log); - } finally { - try { - File.Delete(tempfile); - } catch { } - } - } - }; + [Test] + public void ReadDependencyInput_FullFile1() + { + string[] deps = ReadDependencyInputFromFileData(depFile1, "foo.proto"); + + Assert.NotNull(deps); + Assert.That(deps, Has.Length.InRange(4, 5)); // foo.proto may or may not be listed. + Assert.That(deps, Has.One.EndsWith("wrappers.proto")); + Assert.That(deps, Has.One.EndsWith("type.proto")); + Assert.That(deps, Has.None.StartWith(" ")); + } + + [Test] + public void ReadDependencyInput_FullFile2() + { + string[] deps = ReadDependencyInputFromFileData(depFile2, "C:/projects/foo/src/foo.proto"); + + Assert.NotNull(deps); + Assert.That(deps, Has.Length.InRange(1, 2)); + Assert.That(deps, Has.One.EndsWith("wrappers.proto")); + Assert.That(deps, Has.None.StartWith(" ")); + } + + [Test] + public void ReadDependencyInput_FullFileUnparsable() + { + string[] deps = ReadDependencyInputFromFileData("a:/foo.proto", "/foo.proto"); + Assert.NotNull(deps); + Assert.Zero(deps.Length); + } + + // NB in our tests files are put into the temp directory but all have + // different names. Avoid adding files with the same directory path and + // name, or add reasonable handling for it if required. Tests are run in + // parallel and will collide otherwise. + private string[] ReadDependencyInputFromFileData(string fileData, string protoName) + { + string tempPath = Path.GetTempPath(); + string tempfile = DepFileUtil.GetDepFilenameForProto(tempPath, protoName); + try + { + File.WriteAllText(tempfile, fileData); + var mockEng = new Moq.Mock(); + var log = new TaskLoggingHelper(mockEng.Object, "x"); + return DepFileUtil.ReadDependencyInputs(tempPath, protoName, log); + } + finally + { + try + { + File.Delete(tempfile); + } + catch { } + } + } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/GeneratorTest.cs b/src/csharp/Grpc.Tools.Tests/GeneratorTest.cs index 52fab1d8ca9..8a8fc81aba5 100644 --- a/src/csharp/Grpc.Tools.Tests/GeneratorTest.cs +++ b/src/csharp/Grpc.Tools.Tests/GeneratorTest.cs @@ -21,30 +21,35 @@ using Microsoft.Build.Utilities; using Moq; using NUnit.Framework; -namespace Grpc.Tools.Tests { - public class GeneratorTest { - protected Mock _mockEngine; - protected TaskLoggingHelper _log; +namespace Grpc.Tools.Tests +{ + public class GeneratorTest + { + protected Mock _mockEngine; + protected TaskLoggingHelper _log; - [SetUp] - public void SetUp() { - _mockEngine = new Mock(); - _log = new TaskLoggingHelper(_mockEngine.Object, "dummy"); - } + [SetUp] + public void SetUp() + { + _mockEngine = new Mock(); + _log = new TaskLoggingHelper(_mockEngine.Object, "dummy"); + } - [TestCase("csharp")] - [TestCase("CSharp")] - [TestCase("cpp")] - public void ValidLanguages(string lang) { - Assert.IsNotNull(GeneratorServices.GetForLanguage(lang, _log)); - _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Never); - } + [TestCase("csharp")] + [TestCase("CSharp")] + [TestCase("cpp")] + public void ValidLanguages(string lang) + { + Assert.IsNotNull(GeneratorServices.GetForLanguage(lang, _log)); + _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Never); + } - [TestCase("")] - [TestCase("COBOL")] - public void InvalidLanguages(string lang) { - Assert.IsNull(GeneratorServices.GetForLanguage(lang, _log)); - _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Once); - } - }; + [TestCase("")] + [TestCase("COBOL")] + public void InvalidLanguages(string lang) + { + Assert.IsNull(GeneratorServices.GetForLanguage(lang, _log)); + _mockEngine.Verify(me => me.LogErrorEvent(It.IsAny()), Times.Once); + } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs index 5784cdeac25..418c33820eb 100644 --- a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs +++ b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs @@ -19,13 +19,15 @@ using System.Reflection; using NUnitLite; -namespace Grpc.Tools.Tests { - static class NUnitMain { - public static int Main(string[] args) => +namespace Grpc.Tools.Tests +{ + static class NUnitMain + { + public static int Main(string[] args) => #if NETCOREAPP1_0 || NETCOREAPP1_1 - new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args); + new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args); #else - new AutoRun().Execute(args); + new AutoRun().Execute(args); #endif - }; + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTest.cs index cf9d210424c..ea763f4e408 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileBasicTest.cs @@ -21,50 +21,56 @@ using Microsoft.Build.Framework; using Moq; using NUnit.Framework; -namespace Grpc.Tools.Tests { - public class ProtoCompileBasicTest { - // Mock task class that stops right before invoking protoc. - public class ProtoCompileTestable : ProtoCompile { - public string LastPathToTool { get; private set; } - public string[] LastResponseFile { get; private set; } +namespace Grpc.Tools.Tests +{ + public class ProtoCompileBasicTest + { + // Mock task class that stops right before invoking protoc. + public class ProtoCompileTestable : ProtoCompile + { + public string LastPathToTool { get; private set; } + public string[] LastResponseFile { get; private set; } - protected override int ExecuteTool(string pathToTool, - string response, - string commandLine) { - // We should never be using command line commands. - Assert.That(commandLine, Is.Null | Is.Empty); + protected override int ExecuteTool(string pathToTool, + string response, + string commandLine) + { + // We should never be using command line commands. + Assert.That(commandLine, Is.Null | Is.Empty); - // Must receive a path to tool - Assert.That(pathToTool, Is.Not.Null & Is.Not.Empty); - Assert.That(response, Is.Not.Null & Does.EndWith("\n")); + // Must receive a path to tool + Assert.That(pathToTool, Is.Not.Null & Is.Not.Empty); + Assert.That(response, Is.Not.Null & Does.EndWith("\n")); - LastPathToTool = pathToTool; - LastResponseFile = response.Remove(response.Length - 1).Split('\n'); + LastPathToTool = pathToTool; + LastResponseFile = response.Remove(response.Length - 1).Split('\n'); - // Do not run the tool, but pretend it ran successfully. - return 0; - } - }; + // Do not run the tool, but pretend it ran successfully. + return 0; + } + }; - protected Mock _mockEngine; - protected ProtoCompileTestable _task; + protected Mock _mockEngine; + protected ProtoCompileTestable _task; - [SetUp] - public void SetUp() { - _mockEngine = new Mock(); - _task = new ProtoCompileTestable { - BuildEngine = _mockEngine.Object - }; - } + [SetUp] + public void SetUp() + { + _mockEngine = new Mock(); + _task = new ProtoCompileTestable { + BuildEngine = _mockEngine.Object + }; + } - [TestCase("ProtoBuf")] - [TestCase("Generator")] - [TestCase("OutputDir")] - [Description("We trust MSBuild to initialize these properties.")] - public void RequiredAttributePresentOnProperty(string prop) { - var pinfo = _task.GetType()?.GetProperty(prop); - Assert.NotNull(pinfo); - Assert.That(pinfo, Has.Attribute()); - } - }; + [TestCase("ProtoBuf")] + [TestCase("Generator")] + [TestCase("OutputDir")] + [Description("We trust MSBuild to initialize these properties.")] + public void RequiredAttributePresentOnProperty(string prop) + { + var pinfo = _task.GetType()?.GetProperty(prop); + Assert.NotNull(pinfo); + Assert.That(pinfo, Has.Attribute()); + } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs index 06376f8ef4e..cac71466345 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLineGeneratorTest.cs @@ -21,144 +21,159 @@ using Microsoft.Build.Framework; using Moq; using NUnit.Framework; -namespace Grpc.Tools.Tests { - public class ProtoCompileCommandLineGeneratorTest : ProtoCompileBasicTest { - [SetUp] - public new void SetUp() { - _task.Generator = "csharp"; - _task.OutputDir = "outdir"; - _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); - } - - void ExecuteExpectSuccess() { - _mockEngine - .Setup(me => me.LogErrorEvent(It.IsAny())) - .Callback((BuildErrorEventArgs e) => - Assert.Fail($"Error logged by build engine:\n{e.Message}")); - bool result = _task.Execute(); - Assert.IsTrue(result); - } - - [Test] - public void MinimalCompile() { - ExecuteExpectSuccess(); - Assert.That(_task.LastPathToTool, Does.Match(@"protoc(.exe)?$")); - Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { - "--csharp_out=outdir", "a.proto" })); - } - - [Test] - public void CompileTwoFiles() { - _task.ProtoBuf = Utils.MakeSimpleItems("a.proto", "foo/b.proto"); - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { - "--csharp_out=outdir", "a.proto", "foo/b.proto" })); - } - - [Test] - public void CompileWithProtoPaths() { - _task.ProtoPath = new[] { "/path1", "/path2" }; - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { - "--csharp_out=outdir", "--proto_path=/path1", - "--proto_path=/path2", "a.proto" })); - } - - [TestCase("Cpp")] - [TestCase("CSharp")] - [TestCase("Java")] - [TestCase("Javanano")] - [TestCase("Js")] - [TestCase("Objc")] - [TestCase("Php")] - [TestCase("Python")] - [TestCase("Ruby")] - public void CompileWithOptions(string gen) { - _task.Generator = gen; - _task.OutputOptions = new[] { "foo", "bar" }; - ExecuteExpectSuccess(); - gen = gen.ToLowerInvariant(); - Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { - $"--{gen}_out=outdir", $"--{gen}_opt=foo,bar", "a.proto" })); - } - - [Test] - public void OutputDependencyFile() { - _task.DependencyOut = "foo/my.protodep"; - // Task fails trying to read the non-generated file; we ignore that. - _task.Execute(); - Assert.That(_task.LastResponseFile, - Does.Contain("--dependency_out=foo/my.protodep")); - } - - [Test] - public void OutputDependencyWithProtoDepDir() { - _task.ProtoDepDir = "foo"; - // Task fails trying to read the non-generated file; we ignore that. - _task.Execute(); - Assert.That(_task.LastResponseFile, - Has.One.Match(@"^--dependency_out=foo[/\\].+_a.protodep$")); - } - - [Test] - public void GenerateGrpc() { - _task.GrpcPluginExe = "/foo/grpcgen"; - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { - "--csharp_out=outdir", "--grpc_out=outdir", - "--plugin=protoc-gen-grpc=/foo/grpcgen" })); - } - - [Test] - public void GenerateGrpcWithOutDir() { - _task.GrpcPluginExe = "/foo/grpcgen"; - _task.GrpcOutputDir = "gen-out"; - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { - "--csharp_out=outdir", "--grpc_out=gen-out" })); - } - - [Test] - public void GenerateGrpcWithOptions() { - _task.GrpcPluginExe = "/foo/grpcgen"; - _task.GrpcOutputOptions = new[] { "baz", "quux" }; - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, - Does.Contain("--grpc_opt=baz,quux")); - } - - [Test] - public void DirectoryArgumentsSlashTrimmed() { - _task.GrpcPluginExe = "/foo/grpcgen"; - _task.GrpcOutputDir = "gen-out/"; - _task.OutputDir = "outdir/"; - _task.ProtoPath = new[] { "/path1/", "/path2/" }; - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { +namespace Grpc.Tools.Tests +{ + public class ProtoCompileCommandLineGeneratorTest : ProtoCompileBasicTest + { + [SetUp] + public new void SetUp() + { + _task.Generator = "csharp"; + _task.OutputDir = "outdir"; + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); + } + + void ExecuteExpectSuccess() + { + _mockEngine + .Setup(me => me.LogErrorEvent(It.IsAny())) + .Callback((BuildErrorEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")); + bool result = _task.Execute(); + Assert.IsTrue(result); + } + + [Test] + public void MinimalCompile() + { + ExecuteExpectSuccess(); + Assert.That(_task.LastPathToTool, Does.Match(@"protoc(.exe)?$")); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + "--csharp_out=outdir", "a.proto" })); + } + + [Test] + public void CompileTwoFiles() + { + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto", "foo/b.proto"); + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + "--csharp_out=outdir", "a.proto", "foo/b.proto" })); + } + + [Test] + public void CompileWithProtoPaths() + { + _task.ProtoPath = new[] { "/path1", "/path2" }; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + "--csharp_out=outdir", "--proto_path=/path1", + "--proto_path=/path2", "a.proto" })); + } + + [TestCase("Cpp")] + [TestCase("CSharp")] + [TestCase("Java")] + [TestCase("Javanano")] + [TestCase("Js")] + [TestCase("Objc")] + [TestCase("Php")] + [TestCase("Python")] + [TestCase("Ruby")] + public void CompileWithOptions(string gen) + { + _task.Generator = gen; + _task.OutputOptions = new[] { "foo", "bar" }; + ExecuteExpectSuccess(); + gen = gen.ToLowerInvariant(); + Assert.That(_task.LastResponseFile, Is.EqualTo(new[] { + $"--{gen}_out=outdir", $"--{gen}_opt=foo,bar", "a.proto" })); + } + + [Test] + public void OutputDependencyFile() + { + _task.DependencyOut = "foo/my.protodep"; + // Task fails trying to read the non-generated file; we ignore that. + _task.Execute(); + Assert.That(_task.LastResponseFile, + Does.Contain("--dependency_out=foo/my.protodep")); + } + + [Test] + public void OutputDependencyWithProtoDepDir() + { + _task.ProtoDepDir = "foo"; + // Task fails trying to read the non-generated file; we ignore that. + _task.Execute(); + Assert.That(_task.LastResponseFile, + Has.One.Match(@"^--dependency_out=foo[/\\].+_a.protodep$")); + } + + [Test] + public void GenerateGrpc() + { + _task.GrpcPluginExe = "/foo/grpcgen"; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { + "--csharp_out=outdir", "--grpc_out=outdir", + "--plugin=protoc-gen-grpc=/foo/grpcgen" })); + } + + [Test] + public void GenerateGrpcWithOutDir() + { + _task.GrpcPluginExe = "/foo/grpcgen"; + _task.GrpcOutputDir = "gen-out"; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { + "--csharp_out=outdir", "--grpc_out=gen-out" })); + } + + [Test] + public void GenerateGrpcWithOptions() + { + _task.GrpcPluginExe = "/foo/grpcgen"; + _task.GrpcOutputOptions = new[] { "baz", "quux" }; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, + Does.Contain("--grpc_opt=baz,quux")); + } + + [Test] + public void DirectoryArgumentsSlashTrimmed() + { + _task.GrpcPluginExe = "/foo/grpcgen"; + _task.GrpcOutputDir = "gen-out/"; + _task.OutputDir = "outdir/"; + _task.ProtoPath = new[] { "/path1/", "/path2/" }; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, Is.SupersetOf(new[] { "--proto_path=/path1", "--proto_path=/path2", "--csharp_out=outdir", "--grpc_out=gen-out" })); - } - - [TestCase("." , ".")] - [TestCase("/" , "/")] - [TestCase("//" , "/")] - [TestCase("/foo/" , "/foo")] - [TestCase("/foo" , "/foo")] - [TestCase("foo/" , "foo")] - [TestCase("foo//" , "foo")] - [TestCase("foo/\\" , "foo")] - [TestCase("foo\\/" , "foo")] - [TestCase("C:\\foo", "C:\\foo")] - [TestCase("C:" , "C:")] - [TestCase("C:\\" , "C:\\")] - [TestCase("C:\\\\" , "C:\\")] - public void DirectorySlashTrimmingCases(string given, string expect) { - if (Path.DirectorySeparatorChar == '/') - expect = expect.Replace('\\', '/'); - _task.OutputDir = given; - ExecuteExpectSuccess(); - Assert.That(_task.LastResponseFile, - Does.Contain("--csharp_out=" + expect)); - } - }; + } + + [TestCase(".", ".")] + [TestCase("/", "/")] + [TestCase("//", "/")] + [TestCase("/foo/", "/foo")] + [TestCase("/foo", "/foo")] + [TestCase("foo/", "foo")] + [TestCase("foo//", "foo")] + [TestCase("foo/\\", "foo")] + [TestCase("foo\\/", "foo")] + [TestCase("C:\\foo", "C:\\foo")] + [TestCase("C:", "C:")] + [TestCase("C:\\", "C:\\")] + [TestCase("C:\\\\", "C:\\")] + public void DirectorySlashTrimmingCases(string given, string expect) + { + if (Path.DirectorySeparatorChar == '/') + expect = expect.Replace('\\', '/'); + _task.OutputDir = given; + ExecuteExpectSuccess(); + Assert.That(_task.LastResponseFile, + Does.Contain("--csharp_out=" + expect)); + } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTest.cs index a0406371dcc..1773dcb8750 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoCompileCommandLinePrinterTest.cs @@ -20,28 +20,32 @@ using Microsoft.Build.Framework; using Moq; using NUnit.Framework; -namespace Grpc.Tools.Tests { - public class ProtoCompileCommandLinePrinterTest : ProtoCompileBasicTest { - [SetUp] - public new void SetUp() { - _task.Generator = "csharp"; - _task.OutputDir = "outdir"; - _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); +namespace Grpc.Tools.Tests +{ + public class ProtoCompileCommandLinePrinterTest : ProtoCompileBasicTest + { + [SetUp] + public new void SetUp() + { + _task.Generator = "csharp"; + _task.OutputDir = "outdir"; + _task.ProtoBuf = Utils.MakeSimpleItems("a.proto"); - _mockEngine - .Setup(me => me.LogMessageEvent(It.IsAny())) - .Callback((BuildMessageEventArgs e) => - Assert.Fail($"Error logged by build engine:\n{e.Message}")) - .Verifiable("Command line was not output by the task."); - } + _mockEngine + .Setup(me => me.LogMessageEvent(It.IsAny())) + .Callback((BuildMessageEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")) + .Verifiable("Command line was not output by the task."); + } - void ExecuteExpectSuccess() { - _mockEngine - .Setup(me => me.LogErrorEvent(It.IsAny())) - .Callback((BuildErrorEventArgs e) => - Assert.Fail($"Error logged by build engine:\n{e.Message}")); - bool result = _task.Execute(); - Assert.IsTrue(result); - } - }; + void ExecuteExpectSuccess() + { + _mockEngine + .Setup(me => me.LogErrorEvent(It.IsAny())) + .Callback((BuildErrorEventArgs e) => + Assert.Fail($"Error logged by build engine:\n{e.Message}")); + bool result = _task.Execute(); + Assert.IsTrue(result); + } + }; } diff --git a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs index 2380ae8a37e..54723b74fcb 100644 --- a/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs +++ b/src/csharp/Grpc.Tools.Tests/ProtoToolsPlatformTaskTest.cs @@ -21,102 +21,119 @@ using Microsoft.Build.Framework; using Moq; using NUnit.Framework; -namespace Grpc.Tools.Tests { - public class ProtoToolsPlatformTaskTest { - ProtoToolsPlatform _task; - int _cpuMatched, _osMatched; - - [OneTimeSetUp] - public void SetUp() { - var mockEng = new Mock(); - _task = new ProtoToolsPlatform() { - BuildEngine = mockEng.Object - }; - _task.Execute(); - } - - [OneTimeTearDown] - public void TearDown() { - Assert.AreEqual(1, _cpuMatched, "CPU type detection failed"); - Assert.AreEqual(1, _osMatched, "OS detection failed"); - } +namespace Grpc.Tools.Tests +{ + public class ProtoToolsPlatformTaskTest + { + ProtoToolsPlatform _task; + int _cpuMatched, _osMatched; + + [OneTimeSetUp] + public void SetUp() + { + var mockEng = new Mock(); + _task = new ProtoToolsPlatform() { BuildEngine = mockEng.Object }; + _task.Execute(); + } + + [OneTimeTearDown] + public void TearDown() + { + Assert.AreEqual(1, _cpuMatched, "CPU type detection failed"); + Assert.AreEqual(1, _osMatched, "OS detection failed"); + } #if NETCORE - // PlatformAttribute not yet available in NUnit, coming soon: - // https://github.com/nunit/nunit/pull/3003. - // Use same test case names as under the full framework. - [Test] - public void CpuIsX86() { - if (RuntimeInformation.OSArchitecture == Architecture.X86) { - _cpuMatched++; - Assert.AreEqual("x86", _task.Cpu); - } - } - - [Test] - public void CpuIsX64() { - if (RuntimeInformation.OSArchitecture == Architecture.X64) { - _cpuMatched++; - Assert.AreEqual("x64", _task.Cpu); - } - } - - [Test] - public void OsIsWindows() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - _osMatched++; - Assert.AreEqual("windows", _task.Os); - } - } - - [Test] - public void OsIsLinux() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - _osMatched++; - Assert.AreEqual("linux", _task.Os); - } - } - - [Test] - public void OsIsMacOsX() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - _osMatched++; - Assert.AreEqual("macosx", _task.Os); - } - } + // PlatformAttribute not yet available in NUnit, coming soon: + // https://github.com/nunit/nunit/pull/3003. + // Use same test case names as under the full framework. + [Test] + public void CpuIsX86() + { + if (RuntimeInformation.OSArchitecture == Architecture.X86) + { + _cpuMatched++; + Assert.AreEqual("x86", _task.Cpu); + } + } + + [Test] + public void CpuIsX64() + { + if (RuntimeInformation.OSArchitecture == Architecture.X64) + { + _cpuMatched++; + Assert.AreEqual("x64", _task.Cpu); + } + } + + [Test] + public void OsIsWindows() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _osMatched++; + Assert.AreEqual("windows", _task.Os); + } + } + + [Test] + public void OsIsLinux() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + _osMatched++; + Assert.AreEqual("linux", _task.Os); + } + } + + [Test] + public void OsIsMacOsX() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + _osMatched++; + Assert.AreEqual("macosx", _task.Os); + } + } #else // !NETCORE, i.e. full framework. - [Test, Platform("32-Bit")] - public void CpuIsX86() { - _cpuMatched++; - Assert.AreEqual("x86", _task.Cpu); - } - - [Test, Platform("64-Bit")] - public void CpuIsX64() { - _cpuMatched++; - Assert.AreEqual("x64", _task.Cpu); - } - - [Test, Platform("Win")] - public void OsIsWindows() { - _osMatched++; - Assert.AreEqual("windows", _task.Os); - } - - [Test, Platform("Linux")] - public void OsIsLinux() { - _osMatched++; - Assert.AreEqual("linux", _task.Os); - } - - [Test, Platform("MacOSX")] - public void OsIsMacOsX() { - _osMatched++; - Assert.AreEqual("macosx", _task.Os); - } + [Test, Platform("32-Bit")] + public void CpuIsX86() + { + _cpuMatched++; + Assert.AreEqual("x86", _task.Cpu); + } + + [Test, Platform("64-Bit")] + public void CpuIsX64() + { + _cpuMatched++; + Assert.AreEqual("x64", _task.Cpu); + } + + [Test, Platform("Win")] + public void OsIsWindows() + { + _osMatched++; + Assert.AreEqual("windows", _task.Os); + } + + [Test, Platform("Linux")] + public void OsIsLinux() + { + _osMatched++; + Assert.AreEqual("linux", _task.Os); + } + + [Test, Platform("MacOSX")] + public void OsIsMacOsX() + { + _osMatched++; + Assert.AreEqual("macosx", _task.Os); + } #endif // NETCORE - }; + }; } diff --git a/src/csharp/Grpc.Tools.Tests/Utils.cs b/src/csharp/Grpc.Tools.Tests/Utils.cs index bb051a4873f..6e0f1cffd5e 100644 --- a/src/csharp/Grpc.Tools.Tests/Utils.cs +++ b/src/csharp/Grpc.Tools.Tests/Utils.cs @@ -20,21 +20,27 @@ using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools.Tests { - static class Utils { - // Build an item with a name from args[0] and metadata key-value pairs - // from the rest of args, interleaved. - // This does not do any checking, and expects an odd number of args. - public static ITaskItem MakeItem(params string[] args) { - var item = new TaskItem(args[0]); - for (int i = 1; i < args.Length; i += 2) - item.SetMetadata(args[i], args[i + 1]); - return item; - } +namespace Grpc.Tools.Tests +{ + static class Utils + { + // Build an item with a name from args[0] and metadata key-value pairs + // from the rest of args, interleaved. + // This does not do any checking, and expects an odd number of args. + public static ITaskItem MakeItem(params string[] args) + { + var item = new TaskItem(args[0]); + for (int i = 1; i < args.Length; i += 2) + { + item.SetMetadata(args[i], args[i + 1]); + } + return item; + } - // Return an array of items from given itemspecs. - public static ITaskItem[] MakeSimpleItems(params string[] specs) { - return specs.Select(s => new TaskItem(s)).ToArray(); - } - }; + // Return an array of items from given itemspecs. + public static ITaskItem[] MakeSimpleItems(params string[] specs) + { + return specs.Select(s => new TaskItem(s)).ToArray(); + } + }; } diff --git a/src/csharp/Grpc.Tools/Common.cs b/src/csharp/Grpc.Tools/Common.cs index 9f8600bad0c..e6acdd63939 100644 --- a/src/csharp/Grpc.Tools/Common.cs +++ b/src/csharp/Grpc.Tools/Common.cs @@ -24,82 +24,91 @@ using System.Security; [assembly: InternalsVisibleTo("Grpc.Tools.Tests")] -namespace Grpc.Tools { - // Metadata names (MSBuild item attributes) that we refer to often. - static class Metadata { - // On output dependency lists. - public static string Source = "Source"; - // On ProtoBuf items. - public static string ProtoRoot = "ProtoRoot"; - public static string OutputDir = "OutputDir"; - public static string GrpcServices = "GrpcServices"; - public static string GrpcOutputDir = "GrpcOutputDir"; - }; +namespace Grpc.Tools +{ + // Metadata names (MSBuild item attributes) that we refer to often. + static class Metadata + { + // On output dependency lists. + public static string Source = "Source"; + // On ProtoBuf items. + public static string ProtoRoot = "ProtoRoot"; + public static string OutputDir = "OutputDir"; + public static string GrpcServices = "GrpcServices"; + public static string GrpcOutputDir = "GrpcOutputDir"; + }; - // A few flags used to control the behavior under various platforms. - internal static class Platform { - public enum OsKind { Unknown, Windows, Linux, MacOsX }; - public static readonly OsKind Os; + // A few flags used to control the behavior under various platforms. + internal static class Platform + { + public enum OsKind { Unknown, Windows, Linux, MacOsX }; + public static readonly OsKind Os; - public enum CpuKind { Unknown, X86, X64 }; - public static readonly CpuKind Cpu; + public enum CpuKind { Unknown, X86, X64 }; + public static readonly CpuKind Cpu; - // This is not necessarily true, but good enough. BCL lacks a per-FS - // API to determine file case sensitivity. - public static bool IsFsCaseInsensitive => Os == OsKind.Windows; - public static bool IsWindows => Os == OsKind.Windows; + // This is not necessarily true, but good enough. BCL lacks a per-FS + // API to determine file case sensitivity. + public static bool IsFsCaseInsensitive => Os == OsKind.Windows; + public static bool IsWindows => Os == OsKind.Windows; - static Platform() { + static Platform() + { #if NETCORE - Os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? OsKind.Windows - : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? OsKind.Linux - : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? OsKind.MacOsX - : OsKind.Unknown; + Os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? OsKind.Windows + : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? OsKind.Linux + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? OsKind.MacOsX + : OsKind.Unknown; - switch (RuntimeInformation.OSArchitecture) { - case Architecture.X86: Cpu = CpuKind.X86; break; - case Architecture.X64: Cpu = CpuKind.X64; break; - // We do not have build tools for other architectures. - default: Cpu = CpuKind.Unknown; break; - } + switch (RuntimeInformation.OSArchitecture) + { + case Architecture.X86: Cpu = CpuKind.X86; break; + case Architecture.X64: Cpu = CpuKind.X64; break; + // We do not have build tools for other architectures. + default: Cpu = CpuKind.Unknown; break; + } #else - // Running under either Mono or full MS framework. - Os = OsKind.Windows; - if (Type.GetType("Mono.Runtime", throwOnError: false) != null) { - // Congratulations. We are running under Mono. - var plat = Environment.OSVersion.Platform; - if (plat == PlatformID.MacOSX) { - Os = OsKind.MacOsX; - } else if (plat == PlatformID.Unix || (int)plat == 128) { - // TODO(kkm): This is how Mono detects OSX internally. Looks cheesy - // to me. Would not testing for /proc absence be more reliable? OSX - // did never have it, AFAIK. - Os = File.Exists("/usr/lib/libc.dylib") ? OsKind.MacOsX : OsKind.Linux; - } - } + // Running under either Mono or full MS framework. + Os = OsKind.Windows; + if (Type.GetType("Mono.Runtime", throwOnError: false) != null) + { + // Congratulations. We are running under Mono. + var plat = Environment.OSVersion.Platform; + if (plat == PlatformID.MacOSX) + { + Os = OsKind.MacOsX; + } + else if (plat == PlatformID.Unix || (int)plat == 128) + { + // This is how Mono detects OSX internally. + Os = File.Exists("/usr/lib/libc.dylib") ? OsKind.MacOsX : OsKind.Linux; + } + } - // Hope we are not building on ARM under Xamarin! - Cpu = Environment.Is64BitOperatingSystem ? CpuKind.X64 : CpuKind.X86; + // Hope we are not building on ARM under Xamarin! + Cpu = Environment.Is64BitOperatingSystem ? CpuKind.X64 : CpuKind.X86; #endif - } - }; + } + }; - // Exception handling helpers. - static class Exceptions { - // Returns true iff the exception indicates an error from an I/O call. See - // https://github.com/Microsoft/msbuild/blob/v15.4.8.50001/src/Shared/ExceptionHandling.cs#L101 - static public bool IsIoRelated(Exception ex) => - ex is IOException || - (ex is ArgumentException && !(ex is ArgumentNullException)) || - ex is SecurityException || - ex is UnauthorizedAccessException || - ex is NotSupportedException; - }; + // Exception handling helpers. + static class Exceptions + { + // Returns true iff the exception indicates an error from an I/O call. See + // https://github.com/Microsoft/msbuild/blob/v15.4.8.50001/src/Shared/ExceptionHandling.cs#L101 + static public bool IsIoRelated(Exception ex) => + ex is IOException || + (ex is ArgumentException && !(ex is ArgumentNullException)) || + ex is SecurityException || + ex is UnauthorizedAccessException || + ex is NotSupportedException; + }; - // String helpers. - static class Strings { - // Compare string to argument using OrdinalIgnoreCase comparison. - public static bool EqualNoCase(this string a, string b) => - string.Equals(a, b, StringComparison.OrdinalIgnoreCase); - } + // String helpers. + static class Strings + { + // Compare string to argument using OrdinalIgnoreCase comparison. + public static bool EqualNoCase(this string a, string b) => + string.Equals(a, b, StringComparison.OrdinalIgnoreCase); + } } diff --git a/src/csharp/Grpc.Tools/DepFileUtil.cs b/src/csharp/Grpc.Tools/DepFileUtil.cs index e635ad1e852..440d3d535c8 100644 --- a/src/csharp/Grpc.Tools/DepFileUtil.cs +++ b/src/csharp/Grpc.Tools/DepFileUtil.cs @@ -23,219 +23,251 @@ using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools { - internal static class DepFileUtil { - /* - Sample dependency files. Notable features we have to deal with: - * Slash doubling, must normalize them. - * Spaces in file names. Cannot just "unwrap" the line on backslash at eof; - rather, treat every line as containing one file name except for one with - the ':' separator, as containing exactly two. - * Deal with ':' also being drive letter separator (second example). - - obj\Release\net45\/Foo.cs \ - obj\Release\net45\/FooGrpc.cs: C:/foo/include/google/protobuf/wrappers.proto\ - C:/projects/foo/src//foo.proto - - C:\projects\foo\src\./foo.grpc.pb.cc \ - C:\projects\foo\src\./foo.grpc.pb.h \ - C:\projects\foo\src\./foo.pb.cc \ - C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ - C:/foo/include/google/protobuf/any.proto\ - C:/foo/include/google/protobuf/source_context.proto\ - C:/foo/include/google/protobuf/type.proto\ - foo.proto - */ - - /// - /// Read file names from the dependency file to the right of ':' - /// - /// Relative path to the dependency cache, e. g. "out" - /// Relative path to the proto item, e. g. "foo/file.proto" - /// A for logging - /// - /// Array of the proto file input dependencies as written by protoc, or empty - /// array if the dependency file does not exist or cannot be parsed. - /// - public static string[] ReadDependencyInputs(string protoDepDir, string proto, - TaskLoggingHelper log) { - string depFilename = GetDepFilenameForProto(protoDepDir, proto); - string[] lines = ReadDepFileLines(depFilename, false, log); - if (lines.Length == 0) { - return lines; - } - - var result = new List(); - bool skip = true; - foreach (string line in lines) { - // Start at the only line separating dependency outputs from inputs. - int ix = skip ? FindLineSeparator(line) : -1; - skip = skip && ix < 0; - if (skip) continue; - string file = ExtractFilenameFromLine(line, ix + 1, line.Length); - if (file == "") { - log.LogMessage(MessageImportance.Low, - $"Skipping unparsable dependency file {depFilename}.\nLine with error: '{line}'"); - return new string[0]; +namespace Grpc.Tools +{ + internal static class DepFileUtil + { + /* + Sample dependency files. Notable features we have to deal with: + * Slash doubling, must normalize them. + * Spaces in file names. Cannot just "unwrap" the line on backslash at eof; + rather, treat every line as containing one file name except for one with + the ':' separator, as containing exactly two. + * Deal with ':' also being drive letter separator (second example). + + obj\Release\net45\/Foo.cs \ + obj\Release\net45\/FooGrpc.cs: C:/foo/include/google/protobuf/wrappers.proto\ + C:/projects/foo/src//foo.proto + + C:\projects\foo\src\./foo.grpc.pb.cc \ + C:\projects\foo\src\./foo.grpc.pb.h \ + C:\projects\foo\src\./foo.pb.cc \ + C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ + C:/foo/include/google/protobuf/any.proto\ + C:/foo/include/google/protobuf/source_context.proto\ + C:/foo/include/google/protobuf/type.proto\ + foo.proto + */ + + /// + /// Read file names from the dependency file to the right of ':' + /// + /// Relative path to the dependency cache, e. g. "out" + /// Relative path to the proto item, e. g. "foo/file.proto" + /// A for logging + /// + /// Array of the proto file input dependencies as written by protoc, or empty + /// array if the dependency file does not exist or cannot be parsed. + /// + public static string[] ReadDependencyInputs(string protoDepDir, string proto, + TaskLoggingHelper log) + { + string depFilename = GetDepFilenameForProto(protoDepDir, proto); + string[] lines = ReadDepFileLines(depFilename, false, log); + if (lines.Length == 0) + { + return lines; + } + + var result = new List(); + bool skip = true; + foreach (string line in lines) + { + // Start at the only line separating dependency outputs from inputs. + int ix = skip ? FindLineSeparator(line) : -1; + skip = skip && ix < 0; + if (skip) { continue; } + string file = ExtractFilenameFromLine(line, ix + 1, line.Length); + if (file == "") + { + log.LogMessage(MessageImportance.Low, + $"Skipping unparsable dependency file {depFilename}.\nLine with error: '{line}'"); + return new string[0]; + } + + // Do not bend over backwards trying not to include a proto into its + // own list of dependencies. Since a file is not older than self, + // it is safe to add; this is purely a memory optimization. + if (file != proto) + { + result.Add(file); + } + } + return result.ToArray(); } - // Do not bend over backwards trying not to include a proto into its - // own list of dependencies. Since a file is not older than self, - // it is safe to add; this is purely a memory optimization. - if (file != proto) { - result.Add(file); + /// + /// Read file names from the dependency file to the left of ':' + /// + /// Path to dependency file written by protoc + /// A for logging + /// + /// Array of the protoc-generated outputs from the given dependency file + /// written by protoc, or empty array if the file does not exist or cannot + /// be parsed. + /// + /// + /// Since this is called after a protoc invocation, an unparsable or missing + /// file causes an error-level message to be logged. + /// + public static string[] ReadDependencyOutputs(string depFilename, + TaskLoggingHelper log) + { + string[] lines = ReadDepFileLines(depFilename, true, log); + if (lines.Length == 0) + { + return lines; + } + + var result = new List(); + foreach (string line in lines) + { + int ix = FindLineSeparator(line); + string file = ExtractFilenameFromLine(line, 0, ix >= 0 ? ix : line.Length); + if (file == "") + { + log.LogError("Unable to parse generated dependency file {0}.\n" + + "Line with error: '{1}'", depFilename, line); + return new string[0]; + } + result.Add(file); + + // If this is the line with the separator, do not read further. + if (ix >= 0) { break; } + } + return result.ToArray(); } - } - return result.ToArray(); - } - - /// - /// Read file names from the dependency file to the left of ':' - /// - /// Path to dependency file written by protoc - /// A for logging - /// - /// Array of the protoc-generated outputs from the given dependency file - /// written by protoc, or empty array if the file does not exist or cannot - /// be parsed. - /// - /// - /// Since this is called after a protoc invocation, an unparsable or missing - /// file causes an error-level message to be logged. - /// - public static string[] ReadDependencyOutputs(string depFilename, - TaskLoggingHelper log) { - string[] lines = ReadDepFileLines(depFilename, true, log); - if (lines.Length == 0) { - return lines; - } - - var result = new List(); - foreach (string line in lines) { - int ix = FindLineSeparator(line); - string file = ExtractFilenameFromLine(line, 0, ix >= 0 ? ix : line.Length); - if (file == "") { - log.LogError("Unable to parse generated dependency file {0}.\n" + - "Line with error: '{1}'", depFilename, line); - return new string[0]; + + /// + /// Construct relative dependency file name from directory hash and file name + /// + /// Relative path to the dependency cache, e. g. "out" + /// Relative path to the proto item, e. g. "foo/file.proto" + /// + /// Full relative path to the dependency file, e. g. + /// "out/deadbeef12345678_file.protodep" + /// + /// + /// Since a project may contain proto files with the same filename but in different + /// directories, a unique filename for the dependency file is constructed based on the + /// proto file name both name and directory. The directory path can be arbitrary, + /// for example, it can be outside of the project, or an absolute path including + /// a drive letter, or a UNC network path. A name constructed from such a path by, + /// for example, replacing disallowed name characters with an underscore, may well + /// be over filesystem's allowed path length, since it will be located under the + /// project and solution directories, which are also some level deep from the root. + /// Instead of creating long and unwieldy names for these proto sources, we cache + /// the full path of the name without the filename, and append the filename to it, + /// as in e. g. "foo/file.proto" will yield the name "deadbeef12345678_file", where + /// "deadbeef12345678" is a presumed hash value of the string "foo/". This allows + /// the file names be short, unique (up to a hash collision), and still allowing + /// the user to guess their provenance. + /// + public static string GetDepFilenameForProto(string protoDepDir, string proto) + { + string dirname = Path.GetDirectoryName(proto); + if (Platform.IsFsCaseInsensitive) + { + dirname = dirname.ToLowerInvariant(); + } + string dirhash = HashString64Hex(dirname); + string filename = Path.GetFileNameWithoutExtension(proto); + return Path.Combine(protoDepDir, $"{dirhash}_{filename}.protodep"); } - result.Add(file); - - // If this is the line with the separator, do not read further. - if (ix >= 0) - break; - } - return result.ToArray(); - } - - /// - /// Construct relative dependency file name from directory hash and file name - /// - /// Relative path to the dependency cache, e. g. "out" - /// Relative path to the proto item, e. g. "foo/file.proto" - /// - /// Full relative path to the dependency file, e. g. - /// "out/deadbeef12345678_file.protodep" - /// - /// - /// Since a project may contain proto files with the same filename but in different - /// directories, a unique filename for the dependency file is constructed based on the - /// proto file name both name and directory. The directory path can be arbitrary, - /// for example, it can be outside of the project, or an absolute path including - /// a drive letter, or a UNC network path. A name constructed from such a path by, - /// for example, replacing disallowed name characters with an underscore, may well - /// be over filesystem's allowed path length, since it will be located under the - /// project and solution directories, which are also some level deep from the root. - /// Instead of creating long and unwieldy names for these proto sources, we cache - /// the full path of the name without the filename, and append the filename to it, - /// as in e. g. "foo/file.proto" will yield the name "deadbeef12345678_file", where - /// "deadbeef12345678" is a presumed hash value of the string "foo/". This allows - /// the file names be short, unique (up to a hash collision), and still allowing - /// the user to guess their provenance. - /// - public static string GetDepFilenameForProto(string protoDepDir, string proto) { - string dirname = Path.GetDirectoryName(proto); - if (Platform.IsFsCaseInsensitive) { - dirname = dirname.ToLowerInvariant(); - } - string dirhash = HashString64Hex(dirname); - string filename = Path.GetFileNameWithoutExtension(proto); - return Path.Combine(protoDepDir, $"{dirhash}_{filename}.protodep"); - } - - // Get a 64-bit hash for a directory string. We treat it as if it were - // unique, since there are not so many distinct proto paths in a project. - // We take the first 64 bit of the string SHA1. - // Internal for tests access only. - internal static string HashString64Hex(string str) { - using (var sha1 = System.Security.Cryptography.SHA1.Create()) { - byte[] hash = sha1.ComputeHash(Encoding.UTF8.GetBytes(str)); - var hashstr = new StringBuilder(16); - for (int i = 0; i < 8; i++) { - hashstr.Append(hash[i].ToString("x2")); + + // Get a 64-bit hash for a directory string. We treat it as if it were + // unique, since there are not so many distinct proto paths in a project. + // We take the first 64 bit of the string SHA1. + // Internal for tests access only. + internal static string HashString64Hex(string str) + { + using (var sha1 = System.Security.Cryptography.SHA1.Create()) + { + byte[] hash = sha1.ComputeHash(Encoding.UTF8.GetBytes(str)); + var hashstr = new StringBuilder(16); + for (int i = 0; i < 8; i++) + { + hashstr.Append(hash[i].ToString("x2")); + } + return hashstr.ToString(); + } + } + + // Extract filename between 'beg' (inclusive) and 'end' (exclusive) from + // line 'line', skipping over trailing and leading whitespace, and, when + // 'end' is immediately past end of line 'line', also final '\' (used + // as a line continuation token in the dep file). + // Returns an empty string if the filename cannot be extracted. + static string ExtractFilenameFromLine(string line, int beg, int end) + { + while (beg < end && char.IsWhiteSpace(line[beg])) beg++; + if (beg < end && end == line.Length && line[end - 1] == '\\') end--; + while (beg < end && char.IsWhiteSpace(line[end - 1])) end--; + if (beg == end) return ""; + + string filename = line.Substring(beg, end - beg); + try + { + // Normalize file name. + return Path.Combine(Path.GetDirectoryName(filename), Path.GetFileName(filename)); + } + catch (Exception ex) when (Exceptions.IsIoRelated(ex)) + { + return ""; + } } - return hashstr.ToString(); - } - } - - // Extract filename between 'beg' (inclusive) and 'end' (exclusive) from - // line 'line', skipping over trailing and leading whitespace, and, when - // 'end' is immediately past end of line 'line', also final '\' (used - // as a line continuation token in the dep file). - // Returns an empty string if the filename cannot be extracted. - static string ExtractFilenameFromLine(string line, int beg, int end) { - while (beg < end && char.IsWhiteSpace(line[beg])) beg++; - if (beg < end && end == line.Length && line[end - 1] == '\\') end--; - while (beg < end && char.IsWhiteSpace(line[end - 1])) end--; - if (beg == end) return ""; - - string filename = line.Substring(beg, end - beg); - try { - // Normalize file name. - return Path.Combine( - Path.GetDirectoryName(filename), - Path.GetFileName(filename)); - } catch (Exception ex) when (Exceptions.IsIoRelated(ex)) { - return ""; - } - } - - // Finds the index of the ':' separating dependency clauses in the line, - // not taking Windows drive spec into account. Returns the index of the - // separating ':', or -1 if no separator found. - static int FindLineSeparator(string line) { - // Mind this case where the first ':' is not separator: - // C:\foo\bar\.pb.h: C:/protobuf/wrappers.proto\ - int ix = line.IndexOf(':'); - if (ix <= 0 || ix == line.Length - 1 - || (line[ix + 1] != '/' && line[ix + 1] != '\\') - || !char.IsLetter(line[ix - 1])) - return ix; // Not a windows drive: no letter before ':', or no '\' after. - for (int j = ix - 1; --j >= 0;) { - if (!char.IsWhiteSpace(line[j])) - return ix; // Not space or BOL only before "X:/". - } - return line.IndexOf(':', ix + 1); - } - - // Read entire dependency file. The 'required' parameter controls error - // logging behavior in case the file not found. We require this file when - // compiling, but reading it is optional when computing dependencies. - static string[] ReadDepFileLines(string filename, bool required, - TaskLoggingHelper log) { - try { - var result = File.ReadAllLines(filename); - if (!required) - log.LogMessage(MessageImportance.Low, $"Using dependency file {filename}"); - return result; - } catch (Exception ex) when (Exceptions.IsIoRelated(ex)) { - if (required) { - log.LogError($"Unable to load {filename}: {ex.GetType().Name}: {ex.Message}"); - } else { - log.LogMessage(MessageImportance.Low, $"Skipping {filename}: {ex.Message}"); + + // Finds the index of the ':' separating dependency clauses in the line, + // not taking Windows drive spec into account. Returns the index of the + // separating ':', or -1 if no separator found. + static int FindLineSeparator(string line) + { + // Mind this case where the first ':' is not separator: + // C:\foo\bar\.pb.h: C:/protobuf/wrappers.proto\ + int ix = line.IndexOf(':'); + if (ix <= 0 || ix == line.Length - 1 + || (line[ix + 1] != '/' && line[ix + 1] != '\\') + || !char.IsLetter(line[ix - 1])) + { + return ix; // Not a windows drive: no letter before ':', or no '\' after. + } + for (int j = ix - 1; --j >= 0;) + { + if (!char.IsWhiteSpace(line[j])) + { + return ix; // Not space or BOL only before "X:/". + } + } + return line.IndexOf(':', ix + 1); + } + + // Read entire dependency file. The 'required' parameter controls error + // logging behavior in case the file not found. We require this file when + // compiling, but reading it is optional when computing dependencies. + static string[] ReadDepFileLines(string filename, bool required, + TaskLoggingHelper log) + { + try + { + var result = File.ReadAllLines(filename); + if (!required) + { + log.LogMessage(MessageImportance.Low, $"Using dependency file {filename}"); + } + return result; + } + catch (Exception ex) when (Exceptions.IsIoRelated(ex)) + { + if (required) + { + log.LogError($"Unable to load {filename}: {ex.GetType().Name}: {ex.Message}"); + } + else + { + log.LogMessage(MessageImportance.Low, $"Skipping {filename}: {ex.Message}"); + } + return new string[0]; + } } - return new string[0]; - } - } - }; + }; } diff --git a/src/csharp/Grpc.Tools/GeneratorServices.cs b/src/csharp/Grpc.Tools/GeneratorServices.cs index 52bd29a6781..536ec43c836 100644 --- a/src/csharp/Grpc.Tools/GeneratorServices.cs +++ b/src/csharp/Grpc.Tools/GeneratorServices.cs @@ -22,147 +22,173 @@ using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools { - // Abstract class for language-specific analysis behavior, such - // as guessing the generated files the same way protoc does. - internal abstract class GeneratorServices { - protected readonly TaskLoggingHelper Log; - protected GeneratorServices(TaskLoggingHelper log) { - Log = log; - } - - // Obtain a service for the given language (csharp, cpp). - public static GeneratorServices GetForLanguage(string lang, TaskLoggingHelper log) { - if (lang.EqualNoCase("csharp")) - return new CSharpGeneratorServices(log); - if (lang.EqualNoCase("cpp")) - return new CppGeneratorServices(log); - log.LogError("Invalid value '{0}' for task property 'Generator'. " + - "Supported generator languages: CSharp, Cpp.", lang); - return null; - } - - // Guess whether item's metadata suggests gRPC stub generation. - // When "gRPCServices" is not defined, assume gRPC is not used. - // When defined, C# uses "none" to skip gRPC, C++ uses "false", so - // recognize both. Since the value is tightly coupled to the scripts, - // we do not try to validate the value; scripts take care of that. - // It is safe to assume that gRPC is requested for any other value. - protected bool GrpcOutputPossible(ITaskItem proto) { - string gsm = proto.GetMetadata(Metadata.GrpcServices); - return !gsm.EqualNoCase("") && !gsm.EqualNoCase("none") - && !gsm.EqualNoCase("false"); - } - - public abstract string[] GetPossibleOutputs(ITaskItem proto); - }; - - // C# generator services. - internal class CSharpGeneratorServices : GeneratorServices { - public CSharpGeneratorServices(TaskLoggingHelper log) : base(log) {} - - public override string[] GetPossibleOutputs(ITaskItem protoItem) { - bool doGrpc = GrpcOutputPossible(protoItem); - string filename = LowerUnderscoreToUpperCamel( - Path.GetFileNameWithoutExtension(protoItem.ItemSpec)); - - var outputs = new string[doGrpc ? 2 : 1]; - string outdir = protoItem.GetMetadata(Metadata.OutputDir); - string fileStem = Path.Combine(outdir, filename); - outputs[0] = fileStem + ".cs"; - if (doGrpc) { - // Override outdir if kGrpcOutputDir present, default to proto output. - outdir = protoItem.GetMetadata(Metadata.GrpcOutputDir); - if (outdir != "") { - fileStem = Path.Combine(outdir, filename); +namespace Grpc.Tools +{ + // Abstract class for language-specific analysis behavior, such + // as guessing the generated files the same way protoc does. + internal abstract class GeneratorServices + { + protected readonly TaskLoggingHelper Log; + protected GeneratorServices(TaskLoggingHelper log) { Log = log; } + + // Obtain a service for the given language (csharp, cpp). + public static GeneratorServices GetForLanguage(string lang, TaskLoggingHelper log) + { + if (lang.EqualNoCase("csharp")) { return new CSharpGeneratorServices(log); } + if (lang.EqualNoCase("cpp")) { return new CppGeneratorServices(log); } + + log.LogError("Invalid value '{0}' for task property 'Generator'. " + + "Supported generator languages: CSharp, Cpp.", lang); + return null; } - outputs[1] = fileStem + "Grpc.cs"; - } - return outputs; - } - - string LowerUnderscoreToUpperCamel(string str) { - // See src/compiler/generator_helpers.h:118 - var result = new StringBuilder(str.Length, str.Length); - bool cap = true; - foreach (char c in str) { - if (c == '_') { - cap = true; - } else if (cap) { - result.Append(char.ToUpperInvariant(c)); - cap = false; - } else { - result.Append(c); + + // Guess whether item's metadata suggests gRPC stub generation. + // When "gRPCServices" is not defined, assume gRPC is not used. + // When defined, C# uses "none" to skip gRPC, C++ uses "false", so + // recognize both. Since the value is tightly coupled to the scripts, + // we do not try to validate the value; scripts take care of that. + // It is safe to assume that gRPC is requested for any other value. + protected bool GrpcOutputPossible(ITaskItem proto) + { + string gsm = proto.GetMetadata(Metadata.GrpcServices); + return !gsm.EqualNoCase("") && !gsm.EqualNoCase("none") + && !gsm.EqualNoCase("false"); + } + + public abstract string[] GetPossibleOutputs(ITaskItem proto); + }; + + // C# generator services. + internal class CSharpGeneratorServices : GeneratorServices + { + public CSharpGeneratorServices(TaskLoggingHelper log) : base(log) { } + + public override string[] GetPossibleOutputs(ITaskItem protoItem) + { + bool doGrpc = GrpcOutputPossible(protoItem); + string filename = LowerUnderscoreToUpperCamel( + Path.GetFileNameWithoutExtension(protoItem.ItemSpec)); + + var outputs = new string[doGrpc ? 2 : 1]; + string outdir = protoItem.GetMetadata(Metadata.OutputDir); + string fileStem = Path.Combine(outdir, filename); + outputs[0] = fileStem + ".cs"; + if (doGrpc) + { + // Override outdir if kGrpcOutputDir present, default to proto output. + outdir = protoItem.GetMetadata(Metadata.GrpcOutputDir); + if (outdir != "") + { + fileStem = Path.Combine(outdir, filename); + } + outputs[1] = fileStem + "Grpc.cs"; + } + return outputs; + } + + string LowerUnderscoreToUpperCamel(string str) + { + // See src/compiler/generator_helpers.h:118 + var result = new StringBuilder(str.Length, str.Length); + bool cap = true; + foreach (char c in str) + { + if (c == '_') + { + cap = true; + } + else if (cap) + { + result.Append(char.ToUpperInvariant(c)); + cap = false; + } + else + { + result.Append(c); + } + } + return result.ToString(); } - } - return result.ToString(); - } - }; - - // C++ generator services. - internal class CppGeneratorServices : GeneratorServices { - public CppGeneratorServices(TaskLoggingHelper log) : base(log) { } - - public override string[] GetPossibleOutputs(ITaskItem protoItem) { - bool doGrpc = GrpcOutputPossible(protoItem); - string root = protoItem.GetMetadata(Metadata.ProtoRoot); - string proto = protoItem.ItemSpec; - string filename = Path.GetFileNameWithoutExtension(proto); - // E. g., ("foo/", "foo/bar/x.proto") => "bar" - string relative = GetRelativeDir(root, proto); - - var outputs = new string[doGrpc ? 4 : 2]; - string outdir = protoItem.GetMetadata(Metadata.OutputDir); - string fileStem = Path.Combine(outdir, relative, filename); - outputs[0] = fileStem + ".pb.cc"; - outputs[1] = fileStem + ".pb.h"; - if (doGrpc) { - // Override outdir if kGrpcOutputDir present, default to proto output. - outdir = protoItem.GetMetadata(Metadata.GrpcOutputDir); - if (outdir != "") { - fileStem = Path.Combine(outdir, relative, filename); + }; + + // C++ generator services. + internal class CppGeneratorServices : GeneratorServices + { + public CppGeneratorServices(TaskLoggingHelper log) : base(log) { } + + public override string[] GetPossibleOutputs(ITaskItem protoItem) + { + bool doGrpc = GrpcOutputPossible(protoItem); + string root = protoItem.GetMetadata(Metadata.ProtoRoot); + string proto = protoItem.ItemSpec; + string filename = Path.GetFileNameWithoutExtension(proto); + // E. g., ("foo/", "foo/bar/x.proto") => "bar" + string relative = GetRelativeDir(root, proto); + + var outputs = new string[doGrpc ? 4 : 2]; + string outdir = protoItem.GetMetadata(Metadata.OutputDir); + string fileStem = Path.Combine(outdir, relative, filename); + outputs[0] = fileStem + ".pb.cc"; + outputs[1] = fileStem + ".pb.h"; + if (doGrpc) + { + // Override outdir if kGrpcOutputDir present, default to proto output. + outdir = protoItem.GetMetadata(Metadata.GrpcOutputDir); + if (outdir != "") + { + fileStem = Path.Combine(outdir, relative, filename); + } + outputs[2] = fileStem + "_grpc.pb.cc"; + outputs[3] = fileStem + "_grpc.pb.h"; + } + return outputs; + } + + // Calculate part of proto path relative to root. Protoc is very picky + // about them matching exactly, so can be we. Expect root be exact prefix + // to proto, minus some slash normalization. + string GetRelativeDir(string root, string proto) + { + string protoDir = Path.GetDirectoryName(proto); + string rootDir = EndWithSlash(Path.GetDirectoryName(EndWithSlash(root))); + if (rootDir == s_dotSlash) + { + // Special case, otherwise we can return "./" instead of "" below! + return protoDir; + } + if (Platform.IsFsCaseInsensitive) + { + protoDir = protoDir.ToLowerInvariant(); + rootDir = rootDir.ToLowerInvariant(); + } + protoDir = EndWithSlash(protoDir); + if (!protoDir.StartsWith(rootDir)) + { + Log.LogWarning("ProtoBuf item '{0}' has the ProtoRoot metadata '{1}' " + + "which is not prefix to its path. Cannot compute relative path.", + proto, root); + return ""; + } + return protoDir.Substring(rootDir.Length); + } + + // './' or '.\', normalized per system. + static string s_dotSlash = "." + Path.DirectorySeparatorChar; + + static string EndWithSlash(string str) + { + if (str == "") + { + return s_dotSlash; + } + else if (str[str.Length - 1] != '\\' && str[str.Length - 1] != '/') + { + return str + Path.DirectorySeparatorChar; + } + else + { + return str; + } } - outputs[2] = fileStem + "_grpc.pb.cc"; - outputs[3] = fileStem + "_grpc.pb.h"; - } - return outputs; - } - - // Calculate part of proto path relative to root. Protoc is very picky - // about them matching exactly, so can be we. Expect root be exact prefix - // to proto, minus some slash normalization. - string GetRelativeDir(string root, string proto) { - string protoDir = Path.GetDirectoryName(proto); - string rootDir = EndWithSlash(Path.GetDirectoryName(EndWithSlash(root))); - if (rootDir == s_dotSlash) { - // Special case, otherwise we can return "./" instead of "" below! - return protoDir; - } - if (Platform.IsFsCaseInsensitive) { - protoDir = protoDir.ToLowerInvariant(); - rootDir = rootDir.ToLowerInvariant(); - } - protoDir = EndWithSlash(protoDir); - if (!protoDir.StartsWith(rootDir)) { - Log.LogWarning("ProtoBuf item '{0}' has the ProtoRoot metadata '{1}' " + - "which is not prefix to its path. Cannot compute relative path.", - proto, root); - return ""; - } - return protoDir.Substring(rootDir.Length); - } - - // './' or '.\', normalized per system. - static string s_dotSlash = "." + Path.DirectorySeparatorChar; - - static string EndWithSlash(string str) { - if (str == "") { - return s_dotSlash; - } else if (str[str.Length - 1] != '\\' && str[str.Length - 1] != '/') { - return str + Path.DirectorySeparatorChar; - } else { - return str; - } - } - }; + }; } diff --git a/src/csharp/Grpc.Tools/ProtoCompile.cs b/src/csharp/Grpc.Tools/ProtoCompile.cs index e77084b1ef0..93608e1ac02 100644 --- a/src/csharp/Grpc.Tools/ProtoCompile.cs +++ b/src/csharp/Grpc.Tools/ProtoCompile.cs @@ -20,390 +20,422 @@ using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools { - /// - /// Run Google proto compiler (protoc). - /// - /// After a successful run, the task reads the dependency file if specified - /// to be saved by the compiler, and returns its output files. - /// - /// This task (unlike PrepareProtoCompile) does not attempt to guess anything - /// about language-specific behavior of protoc, and therefore can be used for - /// any language outputs. - /// - public class ProtoCompile : ToolTask { - /* - - Usage: /home/kkm/work/protobuf/src/.libs/lt-protoc [OPTION] PROTO_FILES - Parse PROTO_FILES and generate output based on the options given: - -IPATH, --proto_path=PATH Specify the directory in which to search for - imports. May be specified multiple times; - directories will be searched in order. If not - given, the current working directory is used. - --version Show version info and exit. - -h, --help Show this text and exit. - --encode=MESSAGE_TYPE Read a text-format message of the given type - from standard input and write it in binary - to standard output. The message type must - be defined in PROTO_FILES or their imports. - --decode=MESSAGE_TYPE Read a binary message of the given type from - standard input and write it in text format - to standard output. The message type must - be defined in PROTO_FILES or their imports. - --decode_raw Read an arbitrary protocol message from - standard input and write the raw tag/value - pairs in text format to standard output. No - PROTO_FILES should be given when using this - flag. - --descriptor_set_in=FILES Specifies a delimited list of FILES - each containing a FileDescriptorSet (a - protocol buffer defined in descriptor.proto). - The FileDescriptor for each of the PROTO_FILES - provided will be loaded from these - FileDescriptorSets. If a FileDescriptor - appears multiple times, the first occurrence - will be used. - -oFILE, Writes a FileDescriptorSet (a protocol buffer, - --descriptor_set_out=FILE defined in descriptor.proto) containing all of - the input files to FILE. - --include_imports When using --descriptor_set_out, also include - all dependencies of the input files in the - set, so that the set is self-contained. - --include_source_info When using --descriptor_set_out, do not strip - SourceCodeInfo from the FileDescriptorProto. - This results in vastly larger descriptors that - include information about the original - location of each decl in the source file as - well as surrounding comments. - --dependency_out=FILE Write a dependency output file in the format - expected by make. This writes the transitive - set of input file paths to FILE - --error_format=FORMAT Set the format in which to print errors. - FORMAT may be 'gcc' (the default) or 'msvs' - (Microsoft Visual Studio format). - --print_free_field_numbers Print the free field numbers of the messages - defined in the given proto files. Groups share - the same field number space with the parent - message. Extension ranges are counted as - occupied fields numbers. - - --plugin=EXECUTABLE Specifies a plugin executable to use. - Normally, protoc searches the PATH for - plugins, but you may specify additional - executables not in the path using this flag. - Additionally, EXECUTABLE may be of the form - NAME=PATH, in which case the given plugin name - is mapped to the given executable even if - the executable's own name differs. - --cpp_out=OUT_DIR Generate C++ header and source. - --csharp_out=OUT_DIR Generate C# source file. - --java_out=OUT_DIR Generate Java source file. - --javanano_out=OUT_DIR Generate Java Nano source file. - --js_out=OUT_DIR Generate JavaScript source. - --objc_out=OUT_DIR Generate Objective C header and source. - --php_out=OUT_DIR Generate PHP source file. - --python_out=OUT_DIR Generate Python source file. - --ruby_out=OUT_DIR Generate Ruby source file. - @ Read options and filenames from file. If a - relative file path is specified, the file - will be searched in the working directory. - The --proto_path option will not affect how - this argument file is searched. Content of - the file will be expanded in the position of - @ as in the argument list. Note - that shell expansion is not applied to the - content of the file (i.e., you cannot use - quotes, wildcards, escapes, commands, etc.). - Each line corresponds to a single argument, - even if it contains spaces. - */ - static string[] s_supportedGenerators = new[] { - "cpp", "csharp", "java", - "javanano", "js", "objc", - "php", "python", "ruby", - }; - - /// - /// Code generator. - /// - [Required] - public string Generator { get; set; } - - /// - /// Protobuf files to compile. - /// - [Required] - public ITaskItem[] ProtoBuf { get; set; } - - /// - /// Directory where protoc dependency files are cached. If provided, dependency - /// output filename is autogenerated from source directory hash and file name. - /// Mutually exclusive with DependencyOut. - /// Switch: --dependency_out (with autogenerated file name). - /// - public string ProtoDepDir { get; set; } - - /// - /// Dependency file full name. Mutually exclusive with ProtoDepDir. - /// Autogenerated file name is available in this property after execution. - /// Switch: --dependency_out. - /// - [Output] - public string DependencyOut { get; set; } - - /// - /// The directories to search for imports. Directories will be searched - /// in order. If not given, the current working directory is used. - /// Switch: --proto_path. - /// - public string[] ProtoPath { get; set; } - - /// - /// Generated code directory. The generator property determines the language. - /// Switch: --GEN-out= (for different generators GEN). - /// - [Required] - public string OutputDir { get; set; } - - /// - /// Codegen options. See also OptionsFromMetadata. - /// Switch: --GEN_out= (for different generators GEN). - /// - public string[] OutputOptions { get; set; } - - /// - /// Full path to the gRPC plugin executable. If specified, gRPC generation - /// is enabled for the files. - /// Switch: --plugin=protoc-gen-grpc= - /// - public string GrpcPluginExe { get; set; } - +namespace Grpc.Tools +{ /// - /// Generated gRPC directory. The generator property determines the - /// language. If gRPC is enabled but this is not given, OutputDir is used. - /// Switch: --grpc_out= + /// Run Google proto compiler (protoc). + /// + /// After a successful run, the task reads the dependency file if specified + /// to be saved by the compiler, and returns its output files. + /// + /// This task (unlike PrepareProtoCompile) does not attempt to guess anything + /// about language-specific behavior of protoc, and therefore can be used for + /// any language outputs. /// - public string GrpcOutputDir { get; set; } - - /// - /// gRPC Codegen options. See also OptionsFromMetadata. - /// --grpc_opt=opt1,opt2=val (comma-separated). - /// - public string[] GrpcOutputOptions { get; set; } + public class ProtoCompile : ToolTask + { + /* + + Usage: /home/kkm/work/protobuf/src/.libs/lt-protoc [OPTION] PROTO_FILES + Parse PROTO_FILES and generate output based on the options given: + -IPATH, --proto_path=PATH Specify the directory in which to search for + imports. May be specified multiple times; + directories will be searched in order. If not + given, the current working directory is used. + --version Show version info and exit. + -h, --help Show this text and exit. + --encode=MESSAGE_TYPE Read a text-format message of the given type + from standard input and write it in binary + to standard output. The message type must + be defined in PROTO_FILES or their imports. + --decode=MESSAGE_TYPE Read a binary message of the given type from + standard input and write it in text format + to standard output. The message type must + be defined in PROTO_FILES or their imports. + --decode_raw Read an arbitrary protocol message from + standard input and write the raw tag/value + pairs in text format to standard output. No + PROTO_FILES should be given when using this + flag. + --descriptor_set_in=FILES Specifies a delimited list of FILES + each containing a FileDescriptorSet (a + protocol buffer defined in descriptor.proto). + The FileDescriptor for each of the PROTO_FILES + provided will be loaded from these + FileDescriptorSets. If a FileDescriptor + appears multiple times, the first occurrence + will be used. + -oFILE, Writes a FileDescriptorSet (a protocol buffer, + --descriptor_set_out=FILE defined in descriptor.proto) containing all of + the input files to FILE. + --include_imports When using --descriptor_set_out, also include + all dependencies of the input files in the + set, so that the set is self-contained. + --include_source_info When using --descriptor_set_out, do not strip + SourceCodeInfo from the FileDescriptorProto. + This results in vastly larger descriptors that + include information about the original + location of each decl in the source file as + well as surrounding comments. + --dependency_out=FILE Write a dependency output file in the format + expected by make. This writes the transitive + set of input file paths to FILE + --error_format=FORMAT Set the format in which to print errors. + FORMAT may be 'gcc' (the default) or 'msvs' + (Microsoft Visual Studio format). + --print_free_field_numbers Print the free field numbers of the messages + defined in the given proto files. Groups share + the same field number space with the parent + message. Extension ranges are counted as + occupied fields numbers. + + --plugin=EXECUTABLE Specifies a plugin executable to use. + Normally, protoc searches the PATH for + plugins, but you may specify additional + executables not in the path using this flag. + Additionally, EXECUTABLE may be of the form + NAME=PATH, in which case the given plugin name + is mapped to the given executable even if + the executable's own name differs. + --cpp_out=OUT_DIR Generate C++ header and source. + --csharp_out=OUT_DIR Generate C# source file. + --java_out=OUT_DIR Generate Java source file. + --javanano_out=OUT_DIR Generate Java Nano source file. + --js_out=OUT_DIR Generate JavaScript source. + --objc_out=OUT_DIR Generate Objective C header and source. + --php_out=OUT_DIR Generate PHP source file. + --python_out=OUT_DIR Generate Python source file. + --ruby_out=OUT_DIR Generate Ruby source file. + @ Read options and filenames from file. If a + relative file path is specified, the file + will be searched in the working directory. + The --proto_path option will not affect how + this argument file is searched. Content of + the file will be expanded in the position of + @ as in the argument list. Note + that shell expansion is not applied to the + content of the file (i.e., you cannot use + quotes, wildcards, escapes, commands, etc.). + Each line corresponds to a single argument, + even if it contains spaces. + */ + static string[] s_supportedGenerators = new[] { "cpp", "csharp", "java", + "javanano", "js", "objc", + "php", "python", "ruby" }; + + /// + /// Code generator. + /// + [Required] + public string Generator { get; set; } + + /// + /// Protobuf files to compile. + /// + [Required] + public ITaskItem[] ProtoBuf { get; set; } + + /// + /// Directory where protoc dependency files are cached. If provided, dependency + /// output filename is autogenerated from source directory hash and file name. + /// Mutually exclusive with DependencyOut. + /// Switch: --dependency_out (with autogenerated file name). + /// + public string ProtoDepDir { get; set; } + + /// + /// Dependency file full name. Mutually exclusive with ProtoDepDir. + /// Autogenerated file name is available in this property after execution. + /// Switch: --dependency_out. + /// + [Output] + public string DependencyOut { get; set; } + + /// + /// The directories to search for imports. Directories will be searched + /// in order. If not given, the current working directory is used. + /// Switch: --proto_path. + /// + public string[] ProtoPath { get; set; } + + /// + /// Generated code directory. The generator property determines the language. + /// Switch: --GEN-out= (for different generators GEN). + /// + [Required] + public string OutputDir { get; set; } + + /// + /// Codegen options. See also OptionsFromMetadata. + /// Switch: --GEN_out= (for different generators GEN). + /// + public string[] OutputOptions { get; set; } + + /// + /// Full path to the gRPC plugin executable. If specified, gRPC generation + /// is enabled for the files. + /// Switch: --plugin=protoc-gen-grpc= + /// + public string GrpcPluginExe { get; set; } + + /// + /// Generated gRPC directory. The generator property determines the + /// language. If gRPC is enabled but this is not given, OutputDir is used. + /// Switch: --grpc_out= + /// + public string GrpcOutputDir { get; set; } + + /// + /// gRPC Codegen options. See also OptionsFromMetadata. + /// --grpc_opt=opt1,opt2=val (comma-separated). + /// + public string[] GrpcOutputOptions { get; set; } + + /// + /// List of files written in addition to generated outputs. Includes a + /// single item for the dependency file if written. + /// + [Output] + public ITaskItem[] AdditionalFileWrites { get; private set; } + + /// + /// List of language files generated by protoc. Empty unless DependencyOut + /// or ProtoDepDir is set, since the file writes are extracted from protoc + /// dependency output file. + /// + [Output] + public ITaskItem[] GeneratedFiles { get; private set; } + + // Hide this property from MSBuild, we should never use a shell script. + private new bool UseCommandProcessor { get; set; } + + protected override string ToolName => Platform.IsWindows ? "protoc.exe" : "protoc"; + + // Since we never try to really locate protoc.exe somehow, just try ToolExe + // as the full tool location. It will be either just protoc[.exe] from + // ToolName above if not set by the user, or a user-supplied full path. The + // base class will then resolve the former using system PATH. + protected override string GenerateFullPathToTool() => ToolExe; + + // Log protoc errors with the High priority (bold white in MsBuild, + // printed with -v:n, and shown in the Output windows in VS). + protected override MessageImportance StandardErrorLoggingImportance => MessageImportance.High; + + // Called by base class to validate arguments and make them consistent. + protected override bool ValidateParameters() + { + // Part of proto command line switches, must be lowercased. + Generator = Generator.ToLowerInvariant(); + if (!System.Array.Exists(s_supportedGenerators, g => g == Generator)) + { + Log.LogError("Invalid value for Generator='{0}'. Supported generators: {1}", + Generator, string.Join(", ", s_supportedGenerators)); + } + + if (ProtoDepDir != null && DependencyOut != null) + { + Log.LogError("Properties ProtoDepDir and DependencyOut may not be both specified"); + } + + if (ProtoBuf.Length > 1 && (ProtoDepDir != null || DependencyOut != null)) + { + Log.LogError("Proto compiler currently allows only one input when " + + "--dependency_out is specified (via ProtoDepDir or DependencyOut). " + + "Tracking issue: https://github.com/google/protobuf/pull/3959"); + } + + // Use ProtoDepDir to autogenerate DependencyOut + if (ProtoDepDir != null) + { + DependencyOut = DepFileUtil.GetDepFilenameForProto(ProtoDepDir, ProtoBuf[0].ItemSpec); + } + + if (GrpcPluginExe == null) + { + GrpcOutputOptions = null; + GrpcOutputDir = null; + } + else if (GrpcOutputDir == null) + { + // Use OutputDir for gRPC output if not specified otherwise by user. + GrpcOutputDir = OutputDir; + } + + return !Log.HasLoggedErrors && base.ValidateParameters(); + } - /// - /// List of files written in addition to generated outputs. Includes a - /// single item for the dependency file if written. - /// - [Output] - public ITaskItem[] AdditionalFileWrites { get; private set; } + // Protoc chokes on BOM, naturally. I would! + static readonly Encoding s_utf8WithoutBom = new UTF8Encoding(false); + protected override Encoding ResponseFileEncoding => s_utf8WithoutBom; + + // Protoc takes one argument per line from the response file, and does not + // require any quoting whatsoever. Otherwise, this is similar to the + // standard CommandLineBuilder + class ProtocResponseFileBuilder + { + StringBuilder _data = new StringBuilder(1000); + public override string ToString() => _data.ToString(); + + // If 'value' is not empty, append '--name=value\n'. + public void AddSwitchMaybe(string name, string value) + { + if (!string.IsNullOrEmpty(value)) + { + _data.Append("--").Append(name).Append("=") + .Append(value).Append('\n'); + } + } + + // Add switch with the 'values' separated by commas, for options. + public void AddSwitchMaybe(string name, string[] values) + { + if (values?.Length > 0) + { + _data.Append("--").Append(name).Append("=") + .Append(string.Join(",", values)).Append('\n'); + } + } + + // Add a positional argument to the file data. + public void AddArg(string arg) + { + _data.Append(arg).Append('\n'); + } + }; - /// - /// List of language files generated by protoc. Empty unless DependencyOut - /// or ProtoDepDir is set, since the file writes are extracted from protoc - /// dependency output file. - /// - [Output] - public ITaskItem[] GeneratedFiles { get; private set; } - - // Hide this property from MSBuild, we should never use a shell script. - private new bool UseCommandProcessor { get; set; } - - protected override string ToolName => - Platform.IsWindows ? "protoc.exe" : "protoc"; - - // Since we never try to really locate protoc.exe somehow, just try ToolExe - // as the full tool location. It will be either just protoc[.exe] from - // ToolName above if not set by the user, or a user-supplied full path. The - // base class will then resolve the former using system PATH. - protected override string GenerateFullPathToTool() => ToolExe; - - // Log protoc errors with the High priority (bold white in MsBuild, - // printed with -v:n, and shown in the Output windows in VS). - protected override MessageImportance StandardErrorLoggingImportance => - MessageImportance.High; - - // Called by base class to validate arguments and make them consistent. - protected override bool ValidateParameters() { - // Part of proto command line switches, must be lowercased. - Generator = Generator.ToLowerInvariant(); - if (!System.Array.Exists(s_supportedGenerators, g => g == Generator)) - Log.LogError("Invalid value for Generator='{0}'. Supported generators: {1}", - Generator, string.Join(", ", s_supportedGenerators)); - - if (ProtoDepDir != null && DependencyOut != null) - Log.LogError("Properties ProtoDepDir and DependencyOut may not be both specified"); - - if (ProtoBuf.Length > 1 && (ProtoDepDir != null || DependencyOut != null)) - Log.LogError("Proto compiler currently allows only one input when " + - "--dependency_out is specified (via ProtoDepDir or DependencyOut). " + - "Tracking issue: https://github.com/google/protobuf/pull/3959"); - - // Use ProtoDepDir to autogenerate DependencyOut - if (ProtoDepDir != null) { - DependencyOut = DepFileUtil.GetDepFilenameForProto(ProtoDepDir, ProtoBuf[0].ItemSpec); - } - - if (GrpcPluginExe == null) { - GrpcOutputOptions = null; - GrpcOutputDir = null; - } else if (GrpcOutputDir == null) { - // Use OutputDir for gRPC output if not specified otherwise by user. - GrpcOutputDir = OutputDir; - } - - return !Log.HasLoggedErrors && base.ValidateParameters(); - } - - // Protoc chokes on BOM, naturally. I would! - static readonly Encoding s_utf8WithoutBom = new UTF8Encoding(false); - protected override Encoding ResponseFileEncoding => s_utf8WithoutBom; - - // Protoc takes one argument per line from the response file, and does not - // require any quoting whatsoever. Otherwise, this is similar to the - // standard CommandLineBuilder - class ProtocResponseFileBuilder { - StringBuilder _data = new StringBuilder(1000); - public override string ToString() => _data.ToString(); - - // If 'value' is not empty, append '--name=value\n'. - public void AddSwitchMaybe(string name, string value) { - if (!string.IsNullOrEmpty(value)) { - _data.Append("--").Append(name).Append("=") - .Append(value).Append('\n'); + // Called by the base ToolTask to get response file contents. + protected override string GenerateResponseFileCommands() + { + var cmd = new ProtocResponseFileBuilder(); + cmd.AddSwitchMaybe(Generator + "_out", TrimEndSlash(OutputDir)); + cmd.AddSwitchMaybe(Generator + "_opt", OutputOptions); + cmd.AddSwitchMaybe("plugin=protoc-gen-grpc", GrpcPluginExe); + cmd.AddSwitchMaybe("grpc_out", TrimEndSlash(GrpcOutputDir)); + cmd.AddSwitchMaybe("grpc_opt", GrpcOutputOptions); + if (ProtoPath != null) + { + foreach (string path in ProtoPath) + cmd.AddSwitchMaybe("proto_path", TrimEndSlash(path)); + } + cmd.AddSwitchMaybe("dependency_out", DependencyOut); + foreach (var proto in ProtoBuf) + { + cmd.AddArg(proto.ItemSpec); + } + return cmd.ToString(); } - } - // Add switch with the 'values' separated by commas, for options. - public void AddSwitchMaybe(string name, string[] values) { - if (values?.Length > 0) { - _data.Append("--").Append(name).Append("=") - .Append(string.Join(",", values)).Append('\n'); + // Protoc cannot digest trailing slashes in directory names, + // curiously under Linux, but not in Windows. + static string TrimEndSlash(string dir) + { + if (dir == null || dir.Length <= 1) + { + return dir; + } + string trim = dir.TrimEnd('/', '\\'); + // Do not trim the root slash, drive letter possible. + if (trim.Length == 0) + { + // Slashes all the way down. + return dir.Substring(0, 1); + } + if (trim.Length == 2 && dir.Length > 2 && trim[1] == ':') + { + // We have a drive letter and root, e. g. 'C:\' + return dir.Substring(0, 3); + } + return trim; } - } - - // Add a positional argument to the file data. - public void AddArg(string arg) { - _data.Append(arg).Append('\n'); - } - }; - // Called by the base ToolTask to get response file contents. - protected override string GenerateResponseFileCommands() { - var cmd = new ProtocResponseFileBuilder(); - cmd.AddSwitchMaybe(Generator + "_out", TrimEndSlash(OutputDir)); - cmd.AddSwitchMaybe(Generator + "_opt", OutputOptions); - cmd.AddSwitchMaybe("plugin=protoc-gen-grpc", GrpcPluginExe); - cmd.AddSwitchMaybe("grpc_out", TrimEndSlash(GrpcOutputDir)); - cmd.AddSwitchMaybe("grpc_opt", GrpcOutputOptions); - if (ProtoPath != null) { - foreach (string path in ProtoPath) - cmd.AddSwitchMaybe("proto_path", TrimEndSlash(path)); - } - cmd.AddSwitchMaybe("dependency_out", DependencyOut); - foreach (var proto in ProtoBuf) { - cmd.AddArg(proto.ItemSpec); - } - return cmd.ToString(); - } - - // Protoc cannot digest trailing slashes in directory names, - // curiously under Linux, but not in Windows. - static string TrimEndSlash(string dir) { - if (dir == null || dir.Length <= 1) { - return dir; - } - string trim = dir.TrimEnd('/', '\\'); - // Do not trim the root slash, drive letter possible. - if (trim.Length == 0) { - // Slashes all the way down. - return dir.Substring(0, 1); - } - if (trim.Length == 2 && dir.Length > 2 && trim[1] == ':') { - // We have a drive letter and root, e. g. 'C:\' - return dir.Substring(0, 3); - } - return trim; - } - - // Called by the base class to log tool's command line. - // - // Protoc command file is peculiar, with one argument per line, separated - // by newlines. Unwrap it for log readability into a single line, and also - // quote arguments, lest it look weird and so it may be copied and pasted - // into shell. Since this is for logging only, correct enough is correct. - protected override void LogToolCommand(string cmd) { - var printer = new StringBuilder(1024); - - // Print 'str' slice into 'printer', wrapping in quotes if contains some - // interesting characters in file names, or if empty string. The list of - // characters requiring quoting is not by any means exhaustive; we are - // just striving to be nice, not guaranteeing to be nice. - var quotable = new[] { ' ', '!', '$', '&', '\'', '^' }; - void PrintQuoting(string str, int start, int count) { - bool wrap = count == 0 || str.IndexOfAny(quotable, start, count) >= 0; - if (wrap) printer.Append('"'); - printer.Append(str, start, count); - if (wrap) printer.Append('"'); - } - - for (int ib = 0, ie; (ie = cmd.IndexOf('\n', ib)) >= 0; ib = ie + 1) { - // First line only contains both the program name and the first switch. - // We can rely on at least the '--out_dir' switch being always present. - if (ib == 0) { - int iep = cmd.IndexOf(" --"); - if (iep > 0) { - PrintQuoting(cmd, 0, iep); - ib = iep + 1; - } - } - printer.Append(' '); - if (cmd[ib] == '-') { - // Print switch unquoted, including '=' if any. - int iarg = cmd.IndexOf('=', ib, ie - ib); - if (iarg < 0) { - // Bare switch without a '='. - printer.Append(cmd, ib, ie - ib); - continue; - } - printer.Append(cmd, ib, iarg + 1 - ib); - ib = iarg + 1; - } - // A positional argument or switch value. - PrintQuoting(cmd, ib, ie - ib); - } - - base.LogToolCommand(printer.ToString()); - } - - // Main task entry point. - public override bool Execute() { - base.UseCommandProcessor = false; - - bool ok = base.Execute(); - if (!ok) { - return false; - } - - // Read dependency output file from the compiler to retrieve the - // definitive list of created files. Report the dependency file - // itself as having been written to. - if (DependencyOut != null) { - string[] outputs = DepFileUtil.ReadDependencyOutputs(DependencyOut, Log); - if (HasLoggedErrors) { - return false; + // Called by the base class to log tool's command line. + // + // Protoc command file is peculiar, with one argument per line, separated + // by newlines. Unwrap it for log readability into a single line, and also + // quote arguments, lest it look weird and so it may be copied and pasted + // into shell. Since this is for logging only, correct enough is correct. + protected override void LogToolCommand(string cmd) + { + var printer = new StringBuilder(1024); + + // Print 'str' slice into 'printer', wrapping in quotes if contains some + // interesting characters in file names, or if empty string. The list of + // characters requiring quoting is not by any means exhaustive; we are + // just striving to be nice, not guaranteeing to be nice. + var quotable = new[] { ' ', '!', '$', '&', '\'', '^' }; + void PrintQuoting(string str, int start, int count) + { + bool wrap = count == 0 || str.IndexOfAny(quotable, start, count) >= 0; + if (wrap) printer.Append('"'); + printer.Append(str, start, count); + if (wrap) printer.Append('"'); + } + + for (int ib = 0, ie; (ie = cmd.IndexOf('\n', ib)) >= 0; ib = ie + 1) + { + // First line only contains both the program name and the first switch. + // We can rely on at least the '--out_dir' switch being always present. + if (ib == 0) + { + int iep = cmd.IndexOf(" --"); + if (iep > 0) + { + PrintQuoting(cmd, 0, iep); + ib = iep + 1; + } + } + printer.Append(' '); + if (cmd[ib] == '-') + { + // Print switch unquoted, including '=' if any. + int iarg = cmd.IndexOf('=', ib, ie - ib); + if (iarg < 0) + { + // Bare switch without a '='. + printer.Append(cmd, ib, ie - ib); + continue; + } + printer.Append(cmd, ib, iarg + 1 - ib); + ib = iarg + 1; + } + // A positional argument or switch value. + PrintQuoting(cmd, ib, ie - ib); + } + + base.LogToolCommand(printer.ToString()); } - GeneratedFiles = new ITaskItem[outputs.Length]; - for (int i = 0; i < outputs.Length; i++) { - GeneratedFiles[i] = new TaskItem(outputs[i]); + // Main task entry point. + public override bool Execute() + { + base.UseCommandProcessor = false; + + bool ok = base.Execute(); + if (!ok) + { + return false; + } + + // Read dependency output file from the compiler to retrieve the + // definitive list of created files. Report the dependency file + // itself as having been written to. + if (DependencyOut != null) + { + string[] outputs = DepFileUtil.ReadDependencyOutputs(DependencyOut, Log); + if (HasLoggedErrors) + { + return false; + } + + GeneratedFiles = new ITaskItem[outputs.Length]; + for (int i = 0; i < outputs.Length; i++) + { + GeneratedFiles[i] = new TaskItem(outputs[i]); + } + AdditionalFileWrites = new ITaskItem[] { new TaskItem(DependencyOut) }; + } + + return true; } - AdditionalFileWrites = new ITaskItem[] { - new TaskItem(DependencyOut) - }; - } - - return true; - } - }; + }; } diff --git a/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs b/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs index 74aaa8bd3d5..915be3421e8 100644 --- a/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs +++ b/src/csharp/Grpc.Tools/ProtoCompilerOutputs.cs @@ -1,4 +1,4 @@ -#region Copyright notice and license +#region Copyright notice and license // Copyright 2018 gRPC authors. // @@ -20,61 +20,67 @@ using System.Collections.Generic; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools { - public class ProtoCompilerOutputs : Task { - /// - /// Code generator. Currently supported are "csharp", "cpp". - /// - [Required] - public string Generator { get; set; } +namespace Grpc.Tools +{ + public class ProtoCompilerOutputs : Task + { + /// + /// Code generator. Currently supported are "csharp", "cpp". + /// + [Required] + public string Generator { get; set; } - /// - /// All Proto files in the project. The task computes possible outputs - /// from these proto files, and returns them in the PossibleOutputs list. - /// Not all of these might be actually produced by protoc; this is dealt - /// with later in the ProtoCompile task which returns the list of - /// files actually produced by the compiler. - /// - [Required] - public ITaskItem[] ProtoBuf { get; set; } + /// + /// All Proto files in the project. The task computes possible outputs + /// from these proto files, and returns them in the PossibleOutputs list. + /// Not all of these might be actually produced by protoc; this is dealt + /// with later in the ProtoCompile task which returns the list of + /// files actually produced by the compiler. + /// + [Required] + public ITaskItem[] ProtoBuf { get; set; } - /// - /// Output items per each potential output. We do not look at existing - /// cached dependency even if they exist, since file may be refactored, - /// affecting whether or not gRPC code file is generated from a given proto. - /// Instead, all potentially possible generated sources are collected. - /// It is a wise idea to generate empty files later for those potentials - /// that are not actually created by protoc, so the dependency checks - /// result in a minimal recompilation. The Protoc task can output the - /// list of files it actually produces, given right combination of its - /// properties. - /// Output items will have the Source metadata set on them: - /// - /// - [Output] - public ITaskItem[] PossibleOutputs { get; private set; } + /// + /// Output items per each potential output. We do not look at existing + /// cached dependency even if they exist, since file may be refactored, + /// affecting whether or not gRPC code file is generated from a given proto. + /// Instead, all potentially possible generated sources are collected. + /// It is a wise idea to generate empty files later for those potentials + /// that are not actually created by protoc, so the dependency checks + /// result in a minimal recompilation. The Protoc task can output the + /// list of files it actually produces, given right combination of its + /// properties. + /// Output items will have the Source metadata set on them: + /// + /// + [Output] + public ITaskItem[] PossibleOutputs { get; private set; } - public override bool Execute() { - var generator = GeneratorServices.GetForLanguage(Generator, Log); - if (generator == null) { - // Error already logged, just return. - return false; - } + public override bool Execute() + { + var generator = GeneratorServices.GetForLanguage(Generator, Log); + if (generator == null) + { + // Error already logged, just return. + return false; + } - // Get language-specific possible output. The generator expects certain - // metadata be set on the proto item. - var possible = new List(); - foreach (var proto in ProtoBuf) { - var outputs = generator.GetPossibleOutputs(proto); - foreach (string output in outputs) { - var ti = new TaskItem(output); - ti.SetMetadata(Metadata.Source, proto.ItemSpec); - possible.Add(ti); - } - } - PossibleOutputs = possible.ToArray(); + // Get language-specific possible output. The generator expects certain + // metadata be set on the proto item. + var possible = new List(); + foreach (var proto in ProtoBuf) + { + var outputs = generator.GetPossibleOutputs(proto); + foreach (string output in outputs) + { + var ti = new TaskItem(output); + ti.SetMetadata(Metadata.Source, proto.ItemSpec); + possible.Add(ti); + } + } + PossibleOutputs = possible.ToArray(); - return !Log.HasLoggedErrors; - } - }; + return !Log.HasLoggedErrors; + } + }; } diff --git a/src/csharp/Grpc.Tools/ProtoReadDependencies.cs b/src/csharp/Grpc.Tools/ProtoReadDependencies.cs index ea931b08268..963837e8b74 100644 --- a/src/csharp/Grpc.Tools/ProtoReadDependencies.cs +++ b/src/csharp/Grpc.Tools/ProtoReadDependencies.cs @@ -20,51 +20,59 @@ using System.Collections.Generic; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools { - public class ProtoReadDependencies : Task { - /// - /// The collection is used to collect possible additional dependencies - /// of proto files cached under ProtoDepDir. - /// - [Required] - public ITaskItem[] ProtoBuf { get; set; } +namespace Grpc.Tools +{ + public class ProtoReadDependencies : Task + { + /// + /// The collection is used to collect possible additional dependencies + /// of proto files cached under ProtoDepDir. + /// + [Required] + public ITaskItem[] ProtoBuf { get; set; } - /// - /// Directory where protoc dependency files are cached. - /// - [Required] - public string ProtoDepDir { get; set; } + /// + /// Directory where protoc dependency files are cached. + /// + [Required] + public string ProtoDepDir { get; set; } - /// - /// Additional items that a proto file depends on. This list may include - /// extra dependencies; we do our best to include as few extra positives - /// as reasonable to avoid missing any. The collection item is the - /// dependency, and its Source metadatum is the dependent proto file, like - /// - /// - [Output] - public ITaskItem[] Dependencies { get; private set; } + /// + /// Additional items that a proto file depends on. This list may include + /// extra dependencies; we do our best to include as few extra positives + /// as reasonable to avoid missing any. The collection item is the + /// dependency, and its Source metadatum is the dependent proto file, like + /// + /// + [Output] + public ITaskItem[] Dependencies { get; private set; } - public override bool Execute() { - // Read dependency files, where available. There might be none, - // just use a best effort. - if (ProtoDepDir != null) { - var dependencies = new List(); - foreach (var proto in ProtoBuf) { - string[] deps = DepFileUtil.ReadDependencyInputs(ProtoDepDir, proto.ItemSpec, Log); - foreach (string dep in deps) { - var ti = new TaskItem(dep); - ti.SetMetadata(Metadata.Source, proto.ItemSpec); - dependencies.Add(ti); - } - } - Dependencies = dependencies.ToArray(); - } else { - Dependencies = new ITaskItem[0]; - } + public override bool Execute() + { + // Read dependency files, where available. There might be none, + // just use a best effort. + if (ProtoDepDir != null) + { + var dependencies = new List(); + foreach (var proto in ProtoBuf) + { + string[] deps = DepFileUtil.ReadDependencyInputs(ProtoDepDir, proto.ItemSpec, Log); + foreach (string dep in deps) + { + var ti = new TaskItem(dep); + ti.SetMetadata(Metadata.Source, proto.ItemSpec); + dependencies.Add(ti); + } + } + Dependencies = dependencies.ToArray(); + } + else + { + Dependencies = new ITaskItem[0]; + } - return !Log.HasLoggedErrors; - } - }; + return !Log.HasLoggedErrors; + } + }; } diff --git a/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs b/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs index f505b86fe41..aed8a66339b 100644 --- a/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs +++ b/src/csharp/Grpc.Tools/ProtoToolsPlatform.cs @@ -19,40 +19,45 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -namespace Grpc.Tools { - /// - /// A helper task to resolve actual OS type and bitness. - /// - public class ProtoToolsPlatform : Task { +namespace Grpc.Tools +{ /// - /// Return one of 'linux', 'macosx' or 'windows'. - /// If the OS is unknown, the property is not set. + /// A helper task to resolve actual OS type and bitness. /// - [Output] - public string Os { get; set; } - - /// - /// Return one of 'x64' or 'x86'. - /// If the CPU is unknown, the property is not set. - /// - [Output] - public string Cpu { get; set; } - - - public override bool Execute() { - switch (Platform.Os) { - case Platform.OsKind.Linux: Os = "linux"; break; - case Platform.OsKind.MacOsX: Os = "macosx"; break; - case Platform.OsKind.Windows: Os = "windows"; break; - default: Os = ""; break; - } - - switch (Platform.Cpu) { - case Platform.CpuKind.X86: Cpu = "x86"; break; - case Platform.CpuKind.X64: Cpu = "x64"; break; - default: Cpu = ""; break; - } - return true; - } - }; + public class ProtoToolsPlatform : Task + { + /// + /// Return one of 'linux', 'macosx' or 'windows'. + /// If the OS is unknown, the property is not set. + /// + [Output] + public string Os { get; set; } + + /// + /// Return one of 'x64' or 'x86'. + /// If the CPU is unknown, the property is not set. + /// + [Output] + public string Cpu { get; set; } + + + public override bool Execute() + { + switch (Platform.Os) + { + case Platform.OsKind.Linux: Os = "linux"; break; + case Platform.OsKind.MacOsX: Os = "macosx"; break; + case Platform.OsKind.Windows: Os = "windows"; break; + default: Os = ""; break; + } + + switch (Platform.Cpu) + { + case Platform.CpuKind.X86: Cpu = "x86"; break; + case Platform.CpuKind.X64: Cpu = "x64"; break; + default: Cpu = ""; break; + } + return true; + } + }; } From 0b085a1f05813f54d4164c95fe9431c11f416af3 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 15 Oct 2018 13:11:55 -0700 Subject: [PATCH 20/53] Fix tsan problem at startup. --- src/cpp/server/health/default_health_check_service.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index e86c05c4b0b..4c605310903 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -153,16 +153,18 @@ DefaultHealthCheckService::HealthCheckServiceImpl::~HealthCheckServiceImpl() { } void DefaultHealthCheckService::HealthCheckServiceImpl::StartServingThread() { + // Request the calls we're interested in. + // We do this before starting the serving thread, so that we know it's + // done before server startup is complete. + CheckCallHandler::CreateAndStart(cq_.get(), database_, this); + WatchCallHandler::CreateAndStart(cq_.get(), database_, this); + // Start serving thread. thread_->Start(); } void DefaultHealthCheckService::HealthCheckServiceImpl::Serve(void* arg) { HealthCheckServiceImpl* service = reinterpret_cast(arg); - CheckCallHandler::CreateAndStart(service->cq_.get(), service->database_, - service); - WatchCallHandler::CreateAndStart(service->cq_.get(), service->database_, - service); void* tag; bool ok; while (true) { From a424d96997537ee38ddff153ad7166ac604db7a5 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 15 Oct 2018 14:57:20 -0700 Subject: [PATCH 21/53] clang-format --- src/cpp/server/health/default_health_check_service.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 4c605310903..d7e1356bde2 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -354,10 +354,7 @@ DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: WatchCallHandler(ServerCompletionQueue* cq, DefaultHealthCheckService* database, HealthCheckServiceImpl* service) - : cq_(cq), - database_(database), - service_(service), - stream_(&ctx_) {} + : cq_(cq), database_(database), service_(service), stream_(&ctx_) {} void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: OnCallReceived(std::shared_ptr self, bool ok) { From a464b4ce3971660e81a4a7a79f3ddab6340d16e6 Mon Sep 17 00:00:00 2001 From: "David G. Quintas" Date: Tue, 16 Oct 2018 14:42:37 -0700 Subject: [PATCH 22/53] Revert "grpc_cli json support" --- test/cpp/util/config_grpc_cli.h | 7 - test/cpp/util/grpc_tool.cc | 67 ++--- test/cpp/util/grpc_tool_test.cc | 397 +---------------------------- test/cpp/util/proto_file_parser.cc | 67 ++--- test/cpp/util/proto_file_parser.h | 52 +--- 5 files changed, 57 insertions(+), 533 deletions(-) diff --git a/test/cpp/util/config_grpc_cli.h b/test/cpp/util/config_grpc_cli.h index 1df7b36e2e2..358884196df 100644 --- a/test/cpp/util/config_grpc_cli.h +++ b/test/cpp/util/config_grpc_cli.h @@ -40,11 +40,6 @@ #define GRPC_CUSTOM_TEXTFORMAT ::google::protobuf::TextFormat #endif -#ifndef GRPC_CUSTOM_JSONUTIL -#include -#define GRPC_CUSTOM_JSONUTIL ::google::protobuf::util -#endif - #ifndef GRPC_CUSTOM_DISKSOURCETREE #include #define GRPC_CUSTOM_DISKSOURCETREE ::google::protobuf::compiler::DiskSourceTree @@ -63,8 +58,6 @@ typedef GRPC_CUSTOM_MERGEDDESCRIPTORDATABASE MergedDescriptorDatabase; typedef GRPC_CUSTOM_TEXTFORMAT TextFormat; -namespace json = GRPC_CUSTOM_JSONUTIL; - namespace compiler { typedef GRPC_CUSTOM_DISKSOURCETREE DiskSourceTree; typedef GRPC_CUSTOM_IMPORTER Importer; diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 80eaf4f7279..ccc60cca27a 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -57,8 +57,6 @@ DEFINE_string(proto_path, ".", "Path to look for the proto file."); DEFINE_string(protofiles, "", "Name of the proto file."); DEFINE_bool(binary_input, false, "Input in binary format"); DEFINE_bool(binary_output, false, "Output in binary format"); -DEFINE_bool(json_input, false, "Input in json format"); -DEFINE_bool(json_output, false, "Output in json format"); DEFINE_string(infile, "", "Input file (default is stdin)"); DEFINE_bool(batch, false, "Input contains multiple requests. Please do not use this to send " @@ -90,8 +88,6 @@ class GrpcTool { GrpcToolOutputCallback callback); bool ToText(int argc, const char** argv, const CliCredentials& cred, GrpcToolOutputCallback callback); - bool ToJson(int argc, const char** argv, const CliCredentials& cred, - GrpcToolOutputCallback callback); bool ToBinary(int argc, const char** argv, const CliCredentials& cred, GrpcToolOutputCallback callback); @@ -193,9 +189,8 @@ void ReadResponse(CliCall* call, const grpc::string& method_name, fprintf(stderr, "got response.\n"); if (!FLAGS_binary_output) { gpr_mu_lock(parser_mu); - serialized_response_proto = parser->GetFormattedStringFromMethod( - method_name, serialized_response_proto, false /* is_request */, - FLAGS_json_output); + serialized_response_proto = parser->GetTextFormatFromMethod( + method_name, serialized_response_proto, false /* is_request */); if (parser->HasError() && print_mode) { fprintf(stderr, "Failed to parse response.\n"); } @@ -238,7 +233,6 @@ const Command ops[] = { {"parse", BindWith5Args(&GrpcTool::ParseMessage), 2, 3}, {"totext", BindWith5Args(&GrpcTool::ToText), 2, 3}, {"tobinary", BindWith5Args(&GrpcTool::ToBinary), 2, 3}, - {"tojson", BindWith5Args(&GrpcTool::ToJson), 2, 3}, }; void Usage(const grpc::string& msg) { @@ -250,7 +244,6 @@ void Usage(const grpc::string& msg) { " grpc_cli type ... ; Print type\n" " grpc_cli parse ... ; Parse message\n" " grpc_cli totext ... ; Convert binary message to text\n" - " grpc_cli tojson ... ; Convert binary message to json\n" " grpc_cli tobinary ... ; Convert text message to binary\n" " grpc_cli help ... ; Print this message, or per-command usage\n" "\n", @@ -472,9 +465,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, " --infile ; Input filename (defaults to stdin)\n" " --outfile ; Output filename (defaults to stdout)\n" " --binary_input ; Input in binary format\n" - " --binary_output ; Output in binary format\n" - " --json_input ; Input in json format\n" - " --json_output ; Output in json format\n" + + " --binary_output ; Output in binary format\n" + cred.GetCredentialUsage()); std::stringstream output_ss; @@ -557,8 +548,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, } else { gpr_mu_lock(&parser_mu); serialized_request_proto = parser->GetSerializedProtoFromMethod( - method_name, request_text, true /* is_request */, - FLAGS_json_input); + method_name, request_text, true /* is_request */); request_text.clear(); if (parser->HasError()) { if (print_mode) { @@ -642,8 +632,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, request_text.clear(); } else { serialized_request_proto = parser->GetSerializedProtoFromMethod( - method_name, request_text, true /* is_request */, - FLAGS_json_input); + method_name, request_text, true /* is_request */); request_text.clear(); if (parser->HasError()) { if (print_mode) { @@ -679,10 +668,9 @@ bool GrpcTool::CallMethod(int argc, const char** argv, break; } } else { - grpc::string response_text = parser->GetFormattedStringFromMethod( + grpc::string response_text = parser->GetTextFormatFromMethod( method_name, serialized_response_proto, - false /* is_request */, FLAGS_json_output); - + false /* is_request */); if (parser->HasError() && print_mode) { fprintf(stderr, "Failed to parse response.\n"); } else { @@ -739,7 +727,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, serialized_request_proto = request_text; } else { serialized_request_proto = parser->GetSerializedProtoFromMethod( - method_name, request_text, true /* is_request */, FLAGS_json_input); + method_name, request_text, true /* is_request */); if (parser->HasError()) { fprintf(stderr, "Failed to parse request.\n"); return false; @@ -763,15 +751,13 @@ bool GrpcTool::CallMethod(int argc, const char** argv, receive_initial_metadata ? &server_initial_metadata : nullptr); receive_initial_metadata = false) { if (!FLAGS_binary_output) { - serialized_response_proto = parser->GetFormattedStringFromMethod( - method_name, serialized_response_proto, false /* is_request */, - FLAGS_json_output); + serialized_response_proto = parser->GetTextFormatFromMethod( + method_name, serialized_response_proto, false /* is_request */); if (parser->HasError()) { fprintf(stderr, "Failed to parse response.\n"); return false; } } - if (receive_initial_metadata) { PrintMetadata(server_initial_metadata, "Received initial metadata from server:"); @@ -811,9 +797,7 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, " --infile ; Input filename (defaults to stdin)\n" " --outfile ; Output filename (defaults to stdout)\n" " --binary_input ; Input in binary format\n" - " --binary_output ; Output in binary format\n" - " --json_input ; Input in json format\n" - " --json_output ; Output in json format\n" + + " --binary_output ; Output in binary format\n" + cred.GetCredentialUsage()); std::stringstream output_ss; @@ -860,8 +844,8 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, if (FLAGS_binary_input) { serialized_request_proto = message_text; } else { - serialized_request_proto = parser->GetSerializedProtoFromMessageType( - type_name, message_text, FLAGS_json_input); + serialized_request_proto = + parser->GetSerializedProtoFromMessageType(type_name, message_text); if (parser->HasError()) { fprintf(stderr, "Failed to serialize the message.\n"); return false; @@ -871,14 +855,12 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, if (FLAGS_binary_output) { output_ss << serialized_request_proto; } else { - grpc::string output_text; - output_text = parser->GetFormattedStringFromMessageType( - type_name, serialized_request_proto, FLAGS_json_output); + grpc::string output_text = parser->GetTextFormatFromMessageType( + type_name, serialized_request_proto); if (parser->HasError()) { fprintf(stderr, "Failed to deserialize the message.\n"); return false; } - output_ss << output_text << std::endl; } @@ -903,25 +885,6 @@ bool GrpcTool::ToText(int argc, const char** argv, const CliCredentials& cred, return ParseMessage(argc, argv, cred, callback); } -bool GrpcTool::ToJson(int argc, const char** argv, const CliCredentials& cred, - GrpcToolOutputCallback callback) { - CommandUsage( - "Convert binary message to json\n" - " grpc_cli tojson \n" - " ; Comma separated list of proto files\n" - " ; Protocol buffer type name\n" - " --proto_path ; The search path of proto files\n" - " --infile ; Input filename (defaults to stdin)\n" - " --outfile ; Output filename (defaults to stdout)\n"); - - FLAGS_protofiles = argv[0]; - FLAGS_remotedb = false; - FLAGS_binary_input = true; - FLAGS_binary_output = false; - FLAGS_json_output = true; - return ParseMessage(argc, argv, cred, callback); -} - bool GrpcTool::ToBinary(int argc, const char** argv, const CliCredentials& cred, GrpcToolOutputCallback callback) { CommandUsage( diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index be9a624a2c2..3aae090e818 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -74,20 +74,11 @@ using grpc::testing::EchoResponse; " rpc Echo(grpc.testing.EchoRequest) returns (grpc.testing.EchoResponse) " \ "{}\n" -#define ECHO_RESPONSE_MESSAGE_TEXT_FORMAT \ - "message: \"echo\"\n" \ - "param {\n" \ - " host: \"localhost\"\n" \ - " peer: \"peer\"\n" \ - "}\n\n" - -#define ECHO_RESPONSE_MESSAGE_JSON_FORMAT \ - "{\n" \ - " \"message\": \"echo\",\n" \ - " \"param\": {\n" \ - " \"host\": \"localhost\",\n" \ - " \"peer\": \"peer\"\n" \ - " }\n" \ +#define ECHO_RESPONSE_MESSAGE \ + "message: \"echo\"\n" \ + "param {\n" \ + " host: \"localhost\"\n" \ + " peer: \"peer\"\n" \ "}\n\n" DECLARE_string(channel_creds_type); @@ -98,8 +89,6 @@ namespace testing { DECLARE_bool(binary_input); DECLARE_bool(binary_output); -DECLARE_bool(json_input); -DECLARE_bool(json_output); DECLARE_bool(l); DECLARE_bool(batch); DECLARE_string(metadata); @@ -437,61 +426,6 @@ TEST_F(GrpcToolTest, CallCommand) { // Expected output: "message: \"Hello\"" EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), "message: \"Hello\"")); - - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - - // Expected output: - // { - // "message": "Hello" - // } - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "{\n \"message\": \"Hello\"\n}")); - - ShutdownServer(); -} - -TEST_F(GrpcToolTest, CallCommandJsonInput) { - // Test input "grpc_cli call localhost: Echo "{ \"message\": \"Hello\"}" - std::stringstream output_stream; - - const grpc::string server_address = SetUpServer(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", - "{ \"message\": \"Hello\"}"}; - - FLAGS_json_input = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - // Expected output: "message: \"Hello\"" - EXPECT_TRUE(nullptr != - strstr(output_stream.str().c_str(), "message: \"Hello\"")); - - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - FLAGS_json_input = false; - - // Expected output: - // { - // "message": "Hello" - // } - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "{\n \"message\": \"Hello\"\n}")); - ShutdownServer(); } @@ -519,101 +453,6 @@ TEST_F(GrpcToolTest, CallCommandBatch) { EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), "message: \"Hello0\"\nmessage: " "\"Hello1\"\nmessage: \"Hello2\"\n")); - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - ss.clear(); - ss.seekg(0); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_batch = true; - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - FLAGS_batch = false; - - // Expected output: - // { - // "message": "Hello0" - // } - // { - // "message": "Hello1" - // } - // { - // "message": "Hello2" - // } - // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: - // "Hello2"\n" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "{\n \"message\": \"Hello0\"\n}\n" - "{\n \"message\": \"Hello1\"\n}\n" - "{\n \"message\": \"Hello2\"\n}\n")); - - std::cin.rdbuf(orig); - ShutdownServer(); -} - -TEST_F(GrpcToolTest, CallCommandBatchJsonInput) { - // Test input "grpc_cli call Echo" - std::stringstream output_stream; - - const grpc::string server_address = SetUpServer(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", - "{\"message\": \"Hello0\"}"}; - - // Mock std::cin input "message: 'Hello1'\n\n message: 'Hello2'\n\n" - std::streambuf* orig = std::cin.rdbuf(); - std::istringstream ss( - "{\"message\": \"Hello1\"}\n\n{\"message\": \"Hello2\" }\n\n"); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_json_input = true; - FLAGS_batch = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_batch = false; - - // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: - // "Hello2"\n" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "message: \"Hello0\"\nmessage: " - "\"Hello1\"\nmessage: \"Hello2\"\n")); - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - ss.clear(); - ss.seekg(0); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_batch = true; - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - FLAGS_batch = false; - FLAGS_json_input = false; - - // Expected output: - // { - // "message": "Hello0" - // } - // { - // "message": "Hello1" - // } - // { - // "message": "Hello2" - // } - // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: - // "Hello2"\n" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "{\n \"message\": \"Hello0\"\n}\n" - "{\n \"message\": \"Hello1\"\n}\n" - "{\n \"message\": \"Hello2\"\n}\n")); - std::cin.rdbuf(orig); ShutdownServer(); } @@ -640,95 +479,6 @@ TEST_F(GrpcToolTest, CallCommandBatchWithBadRequest) { // Expected output: "message: "Hello0"\nmessage: "Hello2"\n" EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), "message: \"Hello0\"\nmessage: \"Hello2\"\n")); - - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - ss.clear(); - ss.seekg(0); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_batch = true; - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - FLAGS_batch = false; - - // Expected output: - // { - // "message": "Hello0" - // } - // { - // "message": "Hello2" - // } - // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: - // "Hello2"\n" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "{\n \"message\": \"Hello0\"\n}\n" - "{\n \"message\": \"Hello2\"\n}\n")); - - std::cin.rdbuf(orig); - ShutdownServer(); -} - -TEST_F(GrpcToolTest, CallCommandBatchJsonInputWithBadRequest) { - // Test input "grpc_cli call Echo" - std::stringstream output_stream; - - const grpc::string server_address = SetUpServer(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", - "{ \"message\": \"Hello0\"}"}; - - // Mock std::cin input "message: 1\n\n message: 'Hello2'\n\n" - std::streambuf* orig = std::cin.rdbuf(); - std::istringstream ss( - "{ \"message\": 1 }\n\n { \"message\": \"Hello2\" }\n\n"); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_batch = true; - FLAGS_json_input = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_input = false; - FLAGS_batch = false; - - // Expected output: "message: "Hello0"\nmessage: "Hello2"\n" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "message: \"Hello0\"\nmessage: \"Hello2\"\n")); - - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - ss.clear(); - ss.seekg(0); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_batch = true; - FLAGS_json_input = true; - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - FLAGS_json_input = false; - FLAGS_batch = false; - - // Expected output: - // { - // "message": "Hello0" - // } - // { - // "message": "Hello2" - // } - // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: - // "Hello2"\n" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "{\n \"message\": \"Hello0\"\n}\n" - "{\n \"message\": \"Hello2\"\n}\n")); - std::cin.rdbuf(orig); ShutdownServer(); } @@ -758,34 +508,6 @@ TEST_F(GrpcToolTest, CallCommandRequestStream) { ShutdownServer(); } -TEST_F(GrpcToolTest, CallCommandRequestStreamJsonInput) { - // Test input: grpc_cli call localhost: RequestStream "{ \"message\": - // \"Hello0\"}" - std::stringstream output_stream; - - const grpc::string server_address = SetUpServer(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), - "RequestStream", "{ \"message\": \"Hello0\" }"}; - - // Mock std::cin input "message: 'Hello1'\n\n message: 'Hello2'\n\n" - std::streambuf* orig = std::cin.rdbuf(); - std::istringstream ss( - "{ \"message\": \"Hello1\" }\n\n{ \"message\": \"Hello2\" }\n\n"); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_json_input = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_input = false; - - // Expected output: "message: \"Hello0Hello1Hello2\"" - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - "message: \"Hello0Hello1Hello2\"")); - std::cin.rdbuf(orig); - ShutdownServer(); -} - TEST_F(GrpcToolTest, CallCommandRequestStreamWithBadRequest) { // Test input: grpc_cli call localhost: RequestStream "message: // 'Hello0'" @@ -811,34 +533,6 @@ TEST_F(GrpcToolTest, CallCommandRequestStreamWithBadRequest) { ShutdownServer(); } -TEST_F(GrpcToolTest, CallCommandRequestStreamWithBadRequestJsonInput) { - // Test input: grpc_cli call localhost: RequestStream "message: - // 'Hello0'" - std::stringstream output_stream; - - const grpc::string server_address = SetUpServer(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), - "RequestStream", "{ \"message\": \"Hello0\" }"}; - - // Mock std::cin input "bad_field: 'Hello1'\n\n message: 'Hello2'\n\n" - std::streambuf* orig = std::cin.rdbuf(); - std::istringstream ss( - "{ \"bad_field\": \"Hello1\" }\n\n{ \"message\": \"Hello2\" }\n\n"); - std::cin.rdbuf(ss.rdbuf()); - - FLAGS_json_input = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_input = false; - - // Expected output: "message: \"Hello0Hello2\"" - EXPECT_TRUE(nullptr != - strstr(output_stream.str().c_str(), "message: \"Hello0Hello2\"")); - std::cin.rdbuf(orig); - ShutdownServer(); -} - TEST_F(GrpcToolTest, CallCommandResponseStream) { // Test input: grpc_cli call localhost: ResponseStream "message: // 'Hello'" @@ -860,24 +554,6 @@ TEST_F(GrpcToolTest, CallCommandResponseStream) { expected_response_text.c_str())); } - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - - // Expected output: "{\n \"message\": \"Hello{n}\"\n}\n" - for (int i = 0; i < kServerDefaultResponseStreamsToSend; i++) { - grpc::string expected_response_text = - "{\n \"message\": \"Hello" + grpc::to_string(i) + "\"\n}\n"; - EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), - expected_response_text.c_str())); - } - ShutdownServer(); } @@ -941,31 +617,15 @@ TEST_F(GrpcToolTest, ParseCommand) { const grpc::string server_address = SetUpServer(); const char* argv[] = {"grpc_cli", "parse", server_address.c_str(), - "grpc.testing.EchoResponse", - ECHO_RESPONSE_MESSAGE_TEXT_FORMAT}; + "grpc.testing.EchoResponse", ECHO_RESPONSE_MESSAGE}; FLAGS_binary_input = false; FLAGS_binary_output = false; EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1))); - // Expected output: ECHO_RESPONSE_MESSAGE_TEXT_FORMAT - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), - ECHO_RESPONSE_MESSAGE_TEXT_FORMAT)); - - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - - // Expected output: ECHO_RESPONSE_MESSAGE_JSON_FORMAT - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), - ECHO_RESPONSE_MESSAGE_JSON_FORMAT)); + // Expected output: ECHO_RESPONSE_MESSAGE + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), ECHO_RESPONSE_MESSAGE)); // Parse text message to binary message and then parse it back to text message output_stream.str(grpc::string()); @@ -985,52 +645,13 @@ TEST_F(GrpcToolTest, ParseCommand) { std::placeholders::_1))); // Expected output: ECHO_RESPONSE_MESSAGE - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), - ECHO_RESPONSE_MESSAGE_TEXT_FORMAT)); + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), ECHO_RESPONSE_MESSAGE)); FLAGS_binary_input = false; FLAGS_binary_output = false; ShutdownServer(); } -TEST_F(GrpcToolTest, ParseCommandJsonFormat) { - // Test input "grpc_cli parse localhost: grpc.testing.EchoResponse - // ECHO_RESPONSE_MESSAGE_JSON_FORMAT" - std::stringstream output_stream; - std::stringstream binary_output_stream; - - const grpc::string server_address = SetUpServer(); - const char* argv[] = {"grpc_cli", "parse", server_address.c_str(), - "grpc.testing.EchoResponse", - ECHO_RESPONSE_MESSAGE_JSON_FORMAT}; - - FLAGS_json_input = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - - // Expected output: ECHO_RESPONSE_MESSAGE_TEXT_FORMAT - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), - ECHO_RESPONSE_MESSAGE_TEXT_FORMAT)); - - // with json_output - output_stream.str(grpc::string()); - output_stream.clear(); - - FLAGS_json_output = true; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); - FLAGS_json_output = false; - FLAGS_json_input = false; - - // Expected output: ECHO_RESPONSE_MESSAGE_JSON_FORMAT - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), - ECHO_RESPONSE_MESSAGE_JSON_FORMAT)); - - ShutdownServer(); -} - TEST_F(GrpcToolTest, TooFewArguments) { // Test input "grpc_cli call Echo" std::stringstream output_stream; diff --git a/test/cpp/util/proto_file_parser.cc b/test/cpp/util/proto_file_parser.cc index 68ecfeae2c3..a530ed1ffcb 100644 --- a/test/cpp/util/proto_file_parser.cc +++ b/test/cpp/util/proto_file_parser.cc @@ -217,32 +217,31 @@ bool ProtoFileParser::IsStreaming(const grpc::string& method, bool is_request) { } grpc::string ProtoFileParser::GetSerializedProtoFromMethod( - const grpc::string& method, const grpc::string& formatted_proto, - bool is_request, bool is_json_format) { + const grpc::string& method, const grpc::string& text_format_proto, + bool is_request) { has_error_ = false; grpc::string message_type_name = GetMessageTypeFromMethod(method, is_request); if (has_error_) { return ""; } - return GetSerializedProtoFromMessageType(message_type_name, formatted_proto, - is_json_format); + return GetSerializedProtoFromMessageType(message_type_name, + text_format_proto); } -grpc::string ProtoFileParser::GetFormattedStringFromMethod( +grpc::string ProtoFileParser::GetTextFormatFromMethod( const grpc::string& method, const grpc::string& serialized_proto, - bool is_request, bool is_json_format) { + bool is_request) { has_error_ = false; grpc::string message_type_name = GetMessageTypeFromMethod(method, is_request); if (has_error_) { return ""; } - return GetFormattedStringFromMessageType(message_type_name, serialized_proto, - is_json_format); + return GetTextFormatFromMessageType(message_type_name, serialized_proto); } grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( - const grpc::string& message_type_name, const grpc::string& formatted_proto, - bool is_json_format) { + const grpc::string& message_type_name, + const grpc::string& text_format_proto) { has_error_ = false; grpc::string serialized; const protobuf::Descriptor* desc = @@ -253,23 +252,11 @@ grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( } std::unique_ptr msg( dynamic_factory_->GetPrototype(desc)->New()); - - bool ok; - if (is_json_format) { - ok = grpc::protobuf::json::JsonStringToMessage(formatted_proto, msg.get()) - .ok(); - if (!ok) { - LogError("Failed to convert json format to proto."); - return ""; - } - } else { - ok = protobuf::TextFormat::ParseFromString(formatted_proto, msg.get()); - if (!ok) { - LogError("Failed to convert text format to proto."); - return ""; - } + bool ok = protobuf::TextFormat::ParseFromString(text_format_proto, msg.get()); + if (!ok) { + LogError("Failed to parse text format to proto."); + return ""; } - ok = msg->SerializeToString(&serialized); if (!ok) { LogError("Failed to serialize proto."); @@ -278,9 +265,9 @@ grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( return serialized; } -grpc::string ProtoFileParser::GetFormattedStringFromMessageType( - const grpc::string& message_type_name, const grpc::string& serialized_proto, - bool is_json_format) { +grpc::string ProtoFileParser::GetTextFormatFromMessageType( + const grpc::string& message_type_name, + const grpc::string& serialized_proto) { has_error_ = false; const protobuf::Descriptor* desc = desc_pool_->FindMessageTypeByName(message_type_name); @@ -294,24 +281,12 @@ grpc::string ProtoFileParser::GetFormattedStringFromMessageType( LogError("Failed to deserialize proto."); return ""; } - grpc::string formatted_string; - - if (is_json_format) { - grpc::protobuf::json::JsonPrintOptions jsonPrintOptions; - jsonPrintOptions.add_whitespace = true; - if (!grpc::protobuf::json::MessageToJsonString( - *msg.get(), &formatted_string, jsonPrintOptions) - .ok()) { - LogError("Failed to print proto message to json format"); - return ""; - } - } else { - if (!protobuf::TextFormat::PrintToString(*msg.get(), &formatted_string)) { - LogError("Failed to print proto message to text format"); - return ""; - } + grpc::string text_format; + if (!protobuf::TextFormat::PrintToString(*msg.get(), &text_format)) { + LogError("Failed to print proto message to text format"); + return ""; } - return formatted_string; + return text_format; } void ProtoFileParser::LogError(const grpc::string& error_msg) { diff --git a/test/cpp/util/proto_file_parser.h b/test/cpp/util/proto_file_parser.h index 1e49c98daf9..eb1d793c2ba 100644 --- a/test/cpp/util/proto_file_parser.h +++ b/test/cpp/util/proto_file_parser.h @@ -53,49 +53,21 @@ class ProtoFileParser { // used as the argument of Stub::Call() grpc::string GetFormattedMethodName(const grpc::string& method); - /// Converts a text or json string to its binary proto representation for the - /// given method's input or return type. - /// \param method the name of the method (does not need to be fully qualified - /// name) - /// \param formatted_proto the text- or json-formatted proto string - /// \param is_request if \c true the resolved type is that of the input - /// parameter of the method, otherwise it is the output type - /// \param is_json_format if \c true the \c formatted_proto is treated as a - /// json-formatted proto, otherwise it is treated as a text-formatted - /// proto - /// \return the serialised binary proto represenation of \c formatted_proto - grpc::string GetSerializedProtoFromMethod(const grpc::string& method, - const grpc::string& formatted_proto, - bool is_request, - bool is_json_format); - - /// Converts a text or json string to its proto representation for the given - /// message type. - /// \param formatted_proto the text- or json-formatted proto string - /// \return the serialised binary proto represenation of \c formatted_proto + grpc::string GetSerializedProtoFromMethod( + const grpc::string& method, const grpc::string& text_format_proto, + bool is_request); + + grpc::string GetTextFormatFromMethod(const grpc::string& method, + const grpc::string& serialized_proto, + bool is_request); + grpc::string GetSerializedProtoFromMessageType( const grpc::string& message_type_name, - const grpc::string& formatted_proto, bool is_json_format); - - /// Converts a binary proto string to its text or json string representation - /// for the given method's input or return type. - /// \param method the name of the method (does not need to be a fully - /// qualified name) - /// \param the serialised binary proto representation of type - /// \c message_type_name - /// \return the text- or json-formatted proto string of \c serialized_proto - grpc::string GetFormattedStringFromMethod( - const grpc::string& method, const grpc::string& serialized_proto, - bool is_request, bool is_json_format); - - /// Converts a binary proto string to its text or json string representation - /// for the given message type. - /// \param the serialised binary proto representation of type - /// \c message_type_name - /// \return the text- or json-formatted proto string of \c serialized_proto - grpc::string GetFormattedStringFromMessageType( + const grpc::string& text_format_proto); + + grpc::string GetTextFormatFromMessageType( const grpc::string& message_type_name, - const grpc::string& serialized_proto, bool is_json_format); + const grpc::string& serialized_proto); bool IsStreaming(const grpc::string& method, bool is_request); From 3b7c4082620590d1d0d9419f13f36f2e59291dff Mon Sep 17 00:00:00 2001 From: kkm Date: Tue, 16 Oct 2018 15:27:04 -0700 Subject: [PATCH 23/53] Remove target netcoreapp2.1 from Tools.Tests csproj --- src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index cb065d3c368..a2d4874eece 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -3,7 +3,7 @@ - net45;netcoreapp1.0;netcoreapp2.1 + net45;netcoreapp1.0 Exe From 6bfb96627134a038870ae7106a91b31e289f96b1 Mon Sep 17 00:00:00 2001 From: Yang Gao Date: Tue, 16 Oct 2018 15:49:46 -0700 Subject: [PATCH 24/53] Revert "Revert "grpc_cli json support"" --- test/cpp/util/config_grpc_cli.h | 7 + test/cpp/util/grpc_tool.cc | 67 +++-- test/cpp/util/grpc_tool_test.cc | 397 ++++++++++++++++++++++++++++- test/cpp/util/proto_file_parser.cc | 67 +++-- test/cpp/util/proto_file_parser.h | 52 +++- 5 files changed, 533 insertions(+), 57 deletions(-) diff --git a/test/cpp/util/config_grpc_cli.h b/test/cpp/util/config_grpc_cli.h index 358884196df..1df7b36e2e2 100644 --- a/test/cpp/util/config_grpc_cli.h +++ b/test/cpp/util/config_grpc_cli.h @@ -40,6 +40,11 @@ #define GRPC_CUSTOM_TEXTFORMAT ::google::protobuf::TextFormat #endif +#ifndef GRPC_CUSTOM_JSONUTIL +#include +#define GRPC_CUSTOM_JSONUTIL ::google::protobuf::util +#endif + #ifndef GRPC_CUSTOM_DISKSOURCETREE #include #define GRPC_CUSTOM_DISKSOURCETREE ::google::protobuf::compiler::DiskSourceTree @@ -58,6 +63,8 @@ typedef GRPC_CUSTOM_MERGEDDESCRIPTORDATABASE MergedDescriptorDatabase; typedef GRPC_CUSTOM_TEXTFORMAT TextFormat; +namespace json = GRPC_CUSTOM_JSONUTIL; + namespace compiler { typedef GRPC_CUSTOM_DISKSOURCETREE DiskSourceTree; typedef GRPC_CUSTOM_IMPORTER Importer; diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index ccc60cca27a..80eaf4f7279 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -57,6 +57,8 @@ DEFINE_string(proto_path, ".", "Path to look for the proto file."); DEFINE_string(protofiles, "", "Name of the proto file."); DEFINE_bool(binary_input, false, "Input in binary format"); DEFINE_bool(binary_output, false, "Output in binary format"); +DEFINE_bool(json_input, false, "Input in json format"); +DEFINE_bool(json_output, false, "Output in json format"); DEFINE_string(infile, "", "Input file (default is stdin)"); DEFINE_bool(batch, false, "Input contains multiple requests. Please do not use this to send " @@ -88,6 +90,8 @@ class GrpcTool { GrpcToolOutputCallback callback); bool ToText(int argc, const char** argv, const CliCredentials& cred, GrpcToolOutputCallback callback); + bool ToJson(int argc, const char** argv, const CliCredentials& cred, + GrpcToolOutputCallback callback); bool ToBinary(int argc, const char** argv, const CliCredentials& cred, GrpcToolOutputCallback callback); @@ -189,8 +193,9 @@ void ReadResponse(CliCall* call, const grpc::string& method_name, fprintf(stderr, "got response.\n"); if (!FLAGS_binary_output) { gpr_mu_lock(parser_mu); - serialized_response_proto = parser->GetTextFormatFromMethod( - method_name, serialized_response_proto, false /* is_request */); + serialized_response_proto = parser->GetFormattedStringFromMethod( + method_name, serialized_response_proto, false /* is_request */, + FLAGS_json_output); if (parser->HasError() && print_mode) { fprintf(stderr, "Failed to parse response.\n"); } @@ -233,6 +238,7 @@ const Command ops[] = { {"parse", BindWith5Args(&GrpcTool::ParseMessage), 2, 3}, {"totext", BindWith5Args(&GrpcTool::ToText), 2, 3}, {"tobinary", BindWith5Args(&GrpcTool::ToBinary), 2, 3}, + {"tojson", BindWith5Args(&GrpcTool::ToJson), 2, 3}, }; void Usage(const grpc::string& msg) { @@ -244,6 +250,7 @@ void Usage(const grpc::string& msg) { " grpc_cli type ... ; Print type\n" " grpc_cli parse ... ; Parse message\n" " grpc_cli totext ... ; Convert binary message to text\n" + " grpc_cli tojson ... ; Convert binary message to json\n" " grpc_cli tobinary ... ; Convert text message to binary\n" " grpc_cli help ... ; Print this message, or per-command usage\n" "\n", @@ -465,7 +472,9 @@ bool GrpcTool::CallMethod(int argc, const char** argv, " --infile ; Input filename (defaults to stdin)\n" " --outfile ; Output filename (defaults to stdout)\n" " --binary_input ; Input in binary format\n" - " --binary_output ; Output in binary format\n" + + " --binary_output ; Output in binary format\n" + " --json_input ; Input in json format\n" + " --json_output ; Output in json format\n" + cred.GetCredentialUsage()); std::stringstream output_ss; @@ -548,7 +557,8 @@ bool GrpcTool::CallMethod(int argc, const char** argv, } else { gpr_mu_lock(&parser_mu); serialized_request_proto = parser->GetSerializedProtoFromMethod( - method_name, request_text, true /* is_request */); + method_name, request_text, true /* is_request */, + FLAGS_json_input); request_text.clear(); if (parser->HasError()) { if (print_mode) { @@ -632,7 +642,8 @@ bool GrpcTool::CallMethod(int argc, const char** argv, request_text.clear(); } else { serialized_request_proto = parser->GetSerializedProtoFromMethod( - method_name, request_text, true /* is_request */); + method_name, request_text, true /* is_request */, + FLAGS_json_input); request_text.clear(); if (parser->HasError()) { if (print_mode) { @@ -668,9 +679,10 @@ bool GrpcTool::CallMethod(int argc, const char** argv, break; } } else { - grpc::string response_text = parser->GetTextFormatFromMethod( + grpc::string response_text = parser->GetFormattedStringFromMethod( method_name, serialized_response_proto, - false /* is_request */); + false /* is_request */, FLAGS_json_output); + if (parser->HasError() && print_mode) { fprintf(stderr, "Failed to parse response.\n"); } else { @@ -727,7 +739,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, serialized_request_proto = request_text; } else { serialized_request_proto = parser->GetSerializedProtoFromMethod( - method_name, request_text, true /* is_request */); + method_name, request_text, true /* is_request */, FLAGS_json_input); if (parser->HasError()) { fprintf(stderr, "Failed to parse request.\n"); return false; @@ -751,13 +763,15 @@ bool GrpcTool::CallMethod(int argc, const char** argv, receive_initial_metadata ? &server_initial_metadata : nullptr); receive_initial_metadata = false) { if (!FLAGS_binary_output) { - serialized_response_proto = parser->GetTextFormatFromMethod( - method_name, serialized_response_proto, false /* is_request */); + serialized_response_proto = parser->GetFormattedStringFromMethod( + method_name, serialized_response_proto, false /* is_request */, + FLAGS_json_output); if (parser->HasError()) { fprintf(stderr, "Failed to parse response.\n"); return false; } } + if (receive_initial_metadata) { PrintMetadata(server_initial_metadata, "Received initial metadata from server:"); @@ -797,7 +811,9 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, " --infile ; Input filename (defaults to stdin)\n" " --outfile ; Output filename (defaults to stdout)\n" " --binary_input ; Input in binary format\n" - " --binary_output ; Output in binary format\n" + + " --binary_output ; Output in binary format\n" + " --json_input ; Input in json format\n" + " --json_output ; Output in json format\n" + cred.GetCredentialUsage()); std::stringstream output_ss; @@ -844,8 +860,8 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, if (FLAGS_binary_input) { serialized_request_proto = message_text; } else { - serialized_request_proto = - parser->GetSerializedProtoFromMessageType(type_name, message_text); + serialized_request_proto = parser->GetSerializedProtoFromMessageType( + type_name, message_text, FLAGS_json_input); if (parser->HasError()) { fprintf(stderr, "Failed to serialize the message.\n"); return false; @@ -855,12 +871,14 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, if (FLAGS_binary_output) { output_ss << serialized_request_proto; } else { - grpc::string output_text = parser->GetTextFormatFromMessageType( - type_name, serialized_request_proto); + grpc::string output_text; + output_text = parser->GetFormattedStringFromMessageType( + type_name, serialized_request_proto, FLAGS_json_output); if (parser->HasError()) { fprintf(stderr, "Failed to deserialize the message.\n"); return false; } + output_ss << output_text << std::endl; } @@ -885,6 +903,25 @@ bool GrpcTool::ToText(int argc, const char** argv, const CliCredentials& cred, return ParseMessage(argc, argv, cred, callback); } +bool GrpcTool::ToJson(int argc, const char** argv, const CliCredentials& cred, + GrpcToolOutputCallback callback) { + CommandUsage( + "Convert binary message to json\n" + " grpc_cli tojson \n" + " ; Comma separated list of proto files\n" + " ; Protocol buffer type name\n" + " --proto_path ; The search path of proto files\n" + " --infile ; Input filename (defaults to stdin)\n" + " --outfile ; Output filename (defaults to stdout)\n"); + + FLAGS_protofiles = argv[0]; + FLAGS_remotedb = false; + FLAGS_binary_input = true; + FLAGS_binary_output = false; + FLAGS_json_output = true; + return ParseMessage(argc, argv, cred, callback); +} + bool GrpcTool::ToBinary(int argc, const char** argv, const CliCredentials& cred, GrpcToolOutputCallback callback) { CommandUsage( diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 3aae090e818..be9a624a2c2 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -74,11 +74,20 @@ using grpc::testing::EchoResponse; " rpc Echo(grpc.testing.EchoRequest) returns (grpc.testing.EchoResponse) " \ "{}\n" -#define ECHO_RESPONSE_MESSAGE \ - "message: \"echo\"\n" \ - "param {\n" \ - " host: \"localhost\"\n" \ - " peer: \"peer\"\n" \ +#define ECHO_RESPONSE_MESSAGE_TEXT_FORMAT \ + "message: \"echo\"\n" \ + "param {\n" \ + " host: \"localhost\"\n" \ + " peer: \"peer\"\n" \ + "}\n\n" + +#define ECHO_RESPONSE_MESSAGE_JSON_FORMAT \ + "{\n" \ + " \"message\": \"echo\",\n" \ + " \"param\": {\n" \ + " \"host\": \"localhost\",\n" \ + " \"peer\": \"peer\"\n" \ + " }\n" \ "}\n\n" DECLARE_string(channel_creds_type); @@ -89,6 +98,8 @@ namespace testing { DECLARE_bool(binary_input); DECLARE_bool(binary_output); +DECLARE_bool(json_input); +DECLARE_bool(json_output); DECLARE_bool(l); DECLARE_bool(batch); DECLARE_string(metadata); @@ -426,6 +437,61 @@ TEST_F(GrpcToolTest, CallCommand) { // Expected output: "message: \"Hello\"" EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), "message: \"Hello\"")); + + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + + // Expected output: + // { + // "message": "Hello" + // } + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "{\n \"message\": \"Hello\"\n}")); + + ShutdownServer(); +} + +TEST_F(GrpcToolTest, CallCommandJsonInput) { + // Test input "grpc_cli call localhost: Echo "{ \"message\": \"Hello\"}" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "{ \"message\": \"Hello\"}"}; + + FLAGS_json_input = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + // Expected output: "message: \"Hello\"" + EXPECT_TRUE(nullptr != + strstr(output_stream.str().c_str(), "message: \"Hello\"")); + + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + FLAGS_json_input = false; + + // Expected output: + // { + // "message": "Hello" + // } + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "{\n \"message\": \"Hello\"\n}")); + ShutdownServer(); } @@ -453,6 +519,101 @@ TEST_F(GrpcToolTest, CallCommandBatch) { EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), "message: \"Hello0\"\nmessage: " "\"Hello1\"\nmessage: \"Hello2\"\n")); + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + ss.clear(); + ss.seekg(0); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + FLAGS_batch = false; + + // Expected output: + // { + // "message": "Hello0" + // } + // { + // "message": "Hello1" + // } + // { + // "message": "Hello2" + // } + // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: + // "Hello2"\n" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "{\n \"message\": \"Hello0\"\n}\n" + "{\n \"message\": \"Hello1\"\n}\n" + "{\n \"message\": \"Hello2\"\n}\n")); + + std::cin.rdbuf(orig); + ShutdownServer(); +} + +TEST_F(GrpcToolTest, CallCommandBatchJsonInput) { + // Test input "grpc_cli call Echo" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "{\"message\": \"Hello0\"}"}; + + // Mock std::cin input "message: 'Hello1'\n\n message: 'Hello2'\n\n" + std::streambuf* orig = std::cin.rdbuf(); + std::istringstream ss( + "{\"message\": \"Hello1\"}\n\n{\"message\": \"Hello2\" }\n\n"); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_json_input = true; + FLAGS_batch = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_batch = false; + + // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: + // "Hello2"\n" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "message: \"Hello0\"\nmessage: " + "\"Hello1\"\nmessage: \"Hello2\"\n")); + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + ss.clear(); + ss.seekg(0); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + FLAGS_batch = false; + FLAGS_json_input = false; + + // Expected output: + // { + // "message": "Hello0" + // } + // { + // "message": "Hello1" + // } + // { + // "message": "Hello2" + // } + // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: + // "Hello2"\n" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "{\n \"message\": \"Hello0\"\n}\n" + "{\n \"message\": \"Hello1\"\n}\n" + "{\n \"message\": \"Hello2\"\n}\n")); + std::cin.rdbuf(orig); ShutdownServer(); } @@ -479,6 +640,95 @@ TEST_F(GrpcToolTest, CallCommandBatchWithBadRequest) { // Expected output: "message: "Hello0"\nmessage: "Hello2"\n" EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), "message: \"Hello0\"\nmessage: \"Hello2\"\n")); + + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + ss.clear(); + ss.seekg(0); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + FLAGS_batch = false; + + // Expected output: + // { + // "message": "Hello0" + // } + // { + // "message": "Hello2" + // } + // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: + // "Hello2"\n" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "{\n \"message\": \"Hello0\"\n}\n" + "{\n \"message\": \"Hello2\"\n}\n")); + + std::cin.rdbuf(orig); + ShutdownServer(); +} + +TEST_F(GrpcToolTest, CallCommandBatchJsonInputWithBadRequest) { + // Test input "grpc_cli call Echo" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "{ \"message\": \"Hello0\"}"}; + + // Mock std::cin input "message: 1\n\n message: 'Hello2'\n\n" + std::streambuf* orig = std::cin.rdbuf(); + std::istringstream ss( + "{ \"message\": 1 }\n\n { \"message\": \"Hello2\" }\n\n"); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + FLAGS_json_input = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_input = false; + FLAGS_batch = false; + + // Expected output: "message: "Hello0"\nmessage: "Hello2"\n" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "message: \"Hello0\"\nmessage: \"Hello2\"\n")); + + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + ss.clear(); + ss.seekg(0); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + FLAGS_json_input = true; + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + FLAGS_json_input = false; + FLAGS_batch = false; + + // Expected output: + // { + // "message": "Hello0" + // } + // { + // "message": "Hello2" + // } + // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: + // "Hello2"\n" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "{\n \"message\": \"Hello0\"\n}\n" + "{\n \"message\": \"Hello2\"\n}\n")); + std::cin.rdbuf(orig); ShutdownServer(); } @@ -508,6 +758,34 @@ TEST_F(GrpcToolTest, CallCommandRequestStream) { ShutdownServer(); } +TEST_F(GrpcToolTest, CallCommandRequestStreamJsonInput) { + // Test input: grpc_cli call localhost: RequestStream "{ \"message\": + // \"Hello0\"}" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), + "RequestStream", "{ \"message\": \"Hello0\" }"}; + + // Mock std::cin input "message: 'Hello1'\n\n message: 'Hello2'\n\n" + std::streambuf* orig = std::cin.rdbuf(); + std::istringstream ss( + "{ \"message\": \"Hello1\" }\n\n{ \"message\": \"Hello2\" }\n\n"); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_json_input = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_input = false; + + // Expected output: "message: \"Hello0Hello1Hello2\"" + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + "message: \"Hello0Hello1Hello2\"")); + std::cin.rdbuf(orig); + ShutdownServer(); +} + TEST_F(GrpcToolTest, CallCommandRequestStreamWithBadRequest) { // Test input: grpc_cli call localhost: RequestStream "message: // 'Hello0'" @@ -533,6 +811,34 @@ TEST_F(GrpcToolTest, CallCommandRequestStreamWithBadRequest) { ShutdownServer(); } +TEST_F(GrpcToolTest, CallCommandRequestStreamWithBadRequestJsonInput) { + // Test input: grpc_cli call localhost: RequestStream "message: + // 'Hello0'" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), + "RequestStream", "{ \"message\": \"Hello0\" }"}; + + // Mock std::cin input "bad_field: 'Hello1'\n\n message: 'Hello2'\n\n" + std::streambuf* orig = std::cin.rdbuf(); + std::istringstream ss( + "{ \"bad_field\": \"Hello1\" }\n\n{ \"message\": \"Hello2\" }\n\n"); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_json_input = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_input = false; + + // Expected output: "message: \"Hello0Hello2\"" + EXPECT_TRUE(nullptr != + strstr(output_stream.str().c_str(), "message: \"Hello0Hello2\"")); + std::cin.rdbuf(orig); + ShutdownServer(); +} + TEST_F(GrpcToolTest, CallCommandResponseStream) { // Test input: grpc_cli call localhost: ResponseStream "message: // 'Hello'" @@ -554,6 +860,24 @@ TEST_F(GrpcToolTest, CallCommandResponseStream) { expected_response_text.c_str())); } + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + + // Expected output: "{\n \"message\": \"Hello{n}\"\n}\n" + for (int i = 0; i < kServerDefaultResponseStreamsToSend; i++) { + grpc::string expected_response_text = + "{\n \"message\": \"Hello" + grpc::to_string(i) + "\"\n}\n"; + EXPECT_TRUE(nullptr != strstr(output_stream.str().c_str(), + expected_response_text.c_str())); + } + ShutdownServer(); } @@ -617,15 +941,31 @@ TEST_F(GrpcToolTest, ParseCommand) { const grpc::string server_address = SetUpServer(); const char* argv[] = {"grpc_cli", "parse", server_address.c_str(), - "grpc.testing.EchoResponse", ECHO_RESPONSE_MESSAGE}; + "grpc.testing.EchoResponse", + ECHO_RESPONSE_MESSAGE_TEXT_FORMAT}; FLAGS_binary_input = false; FLAGS_binary_output = false; EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1))); - // Expected output: ECHO_RESPONSE_MESSAGE - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), ECHO_RESPONSE_MESSAGE)); + // Expected output: ECHO_RESPONSE_MESSAGE_TEXT_FORMAT + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + ECHO_RESPONSE_MESSAGE_TEXT_FORMAT)); + + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + + // Expected output: ECHO_RESPONSE_MESSAGE_JSON_FORMAT + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + ECHO_RESPONSE_MESSAGE_JSON_FORMAT)); // Parse text message to binary message and then parse it back to text message output_stream.str(grpc::string()); @@ -645,13 +985,52 @@ TEST_F(GrpcToolTest, ParseCommand) { std::placeholders::_1))); // Expected output: ECHO_RESPONSE_MESSAGE - EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), ECHO_RESPONSE_MESSAGE)); + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + ECHO_RESPONSE_MESSAGE_TEXT_FORMAT)); FLAGS_binary_input = false; FLAGS_binary_output = false; ShutdownServer(); } +TEST_F(GrpcToolTest, ParseCommandJsonFormat) { + // Test input "grpc_cli parse localhost: grpc.testing.EchoResponse + // ECHO_RESPONSE_MESSAGE_JSON_FORMAT" + std::stringstream output_stream; + std::stringstream binary_output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "parse", server_address.c_str(), + "grpc.testing.EchoResponse", + ECHO_RESPONSE_MESSAGE_JSON_FORMAT}; + + FLAGS_json_input = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + + // Expected output: ECHO_RESPONSE_MESSAGE_TEXT_FORMAT + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + ECHO_RESPONSE_MESSAGE_TEXT_FORMAT)); + + // with json_output + output_stream.str(grpc::string()); + output_stream.clear(); + + FLAGS_json_output = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_json_output = false; + FLAGS_json_input = false; + + // Expected output: ECHO_RESPONSE_MESSAGE_JSON_FORMAT + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + ECHO_RESPONSE_MESSAGE_JSON_FORMAT)); + + ShutdownServer(); +} + TEST_F(GrpcToolTest, TooFewArguments) { // Test input "grpc_cli call Echo" std::stringstream output_stream; diff --git a/test/cpp/util/proto_file_parser.cc b/test/cpp/util/proto_file_parser.cc index a530ed1ffcb..68ecfeae2c3 100644 --- a/test/cpp/util/proto_file_parser.cc +++ b/test/cpp/util/proto_file_parser.cc @@ -217,31 +217,32 @@ bool ProtoFileParser::IsStreaming(const grpc::string& method, bool is_request) { } grpc::string ProtoFileParser::GetSerializedProtoFromMethod( - const grpc::string& method, const grpc::string& text_format_proto, - bool is_request) { + const grpc::string& method, const grpc::string& formatted_proto, + bool is_request, bool is_json_format) { has_error_ = false; grpc::string message_type_name = GetMessageTypeFromMethod(method, is_request); if (has_error_) { return ""; } - return GetSerializedProtoFromMessageType(message_type_name, - text_format_proto); + return GetSerializedProtoFromMessageType(message_type_name, formatted_proto, + is_json_format); } -grpc::string ProtoFileParser::GetTextFormatFromMethod( +grpc::string ProtoFileParser::GetFormattedStringFromMethod( const grpc::string& method, const grpc::string& serialized_proto, - bool is_request) { + bool is_request, bool is_json_format) { has_error_ = false; grpc::string message_type_name = GetMessageTypeFromMethod(method, is_request); if (has_error_) { return ""; } - return GetTextFormatFromMessageType(message_type_name, serialized_proto); + return GetFormattedStringFromMessageType(message_type_name, serialized_proto, + is_json_format); } grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( - const grpc::string& message_type_name, - const grpc::string& text_format_proto) { + const grpc::string& message_type_name, const grpc::string& formatted_proto, + bool is_json_format) { has_error_ = false; grpc::string serialized; const protobuf::Descriptor* desc = @@ -252,11 +253,23 @@ grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( } std::unique_ptr msg( dynamic_factory_->GetPrototype(desc)->New()); - bool ok = protobuf::TextFormat::ParseFromString(text_format_proto, msg.get()); - if (!ok) { - LogError("Failed to parse text format to proto."); - return ""; + + bool ok; + if (is_json_format) { + ok = grpc::protobuf::json::JsonStringToMessage(formatted_proto, msg.get()) + .ok(); + if (!ok) { + LogError("Failed to convert json format to proto."); + return ""; + } + } else { + ok = protobuf::TextFormat::ParseFromString(formatted_proto, msg.get()); + if (!ok) { + LogError("Failed to convert text format to proto."); + return ""; + } } + ok = msg->SerializeToString(&serialized); if (!ok) { LogError("Failed to serialize proto."); @@ -265,9 +278,9 @@ grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( return serialized; } -grpc::string ProtoFileParser::GetTextFormatFromMessageType( - const grpc::string& message_type_name, - const grpc::string& serialized_proto) { +grpc::string ProtoFileParser::GetFormattedStringFromMessageType( + const grpc::string& message_type_name, const grpc::string& serialized_proto, + bool is_json_format) { has_error_ = false; const protobuf::Descriptor* desc = desc_pool_->FindMessageTypeByName(message_type_name); @@ -281,12 +294,24 @@ grpc::string ProtoFileParser::GetTextFormatFromMessageType( LogError("Failed to deserialize proto."); return ""; } - grpc::string text_format; - if (!protobuf::TextFormat::PrintToString(*msg.get(), &text_format)) { - LogError("Failed to print proto message to text format"); - return ""; + grpc::string formatted_string; + + if (is_json_format) { + grpc::protobuf::json::JsonPrintOptions jsonPrintOptions; + jsonPrintOptions.add_whitespace = true; + if (!grpc::protobuf::json::MessageToJsonString( + *msg.get(), &formatted_string, jsonPrintOptions) + .ok()) { + LogError("Failed to print proto message to json format"); + return ""; + } + } else { + if (!protobuf::TextFormat::PrintToString(*msg.get(), &formatted_string)) { + LogError("Failed to print proto message to text format"); + return ""; + } } - return text_format; + return formatted_string; } void ProtoFileParser::LogError(const grpc::string& error_msg) { diff --git a/test/cpp/util/proto_file_parser.h b/test/cpp/util/proto_file_parser.h index eb1d793c2ba..1e49c98daf9 100644 --- a/test/cpp/util/proto_file_parser.h +++ b/test/cpp/util/proto_file_parser.h @@ -53,21 +53,49 @@ class ProtoFileParser { // used as the argument of Stub::Call() grpc::string GetFormattedMethodName(const grpc::string& method); - grpc::string GetSerializedProtoFromMethod( - const grpc::string& method, const grpc::string& text_format_proto, - bool is_request); - - grpc::string GetTextFormatFromMethod(const grpc::string& method, - const grpc::string& serialized_proto, - bool is_request); - + /// Converts a text or json string to its binary proto representation for the + /// given method's input or return type. + /// \param method the name of the method (does not need to be fully qualified + /// name) + /// \param formatted_proto the text- or json-formatted proto string + /// \param is_request if \c true the resolved type is that of the input + /// parameter of the method, otherwise it is the output type + /// \param is_json_format if \c true the \c formatted_proto is treated as a + /// json-formatted proto, otherwise it is treated as a text-formatted + /// proto + /// \return the serialised binary proto represenation of \c formatted_proto + grpc::string GetSerializedProtoFromMethod(const grpc::string& method, + const grpc::string& formatted_proto, + bool is_request, + bool is_json_format); + + /// Converts a text or json string to its proto representation for the given + /// message type. + /// \param formatted_proto the text- or json-formatted proto string + /// \return the serialised binary proto represenation of \c formatted_proto grpc::string GetSerializedProtoFromMessageType( const grpc::string& message_type_name, - const grpc::string& text_format_proto); - - grpc::string GetTextFormatFromMessageType( + const grpc::string& formatted_proto, bool is_json_format); + + /// Converts a binary proto string to its text or json string representation + /// for the given method's input or return type. + /// \param method the name of the method (does not need to be a fully + /// qualified name) + /// \param the serialised binary proto representation of type + /// \c message_type_name + /// \return the text- or json-formatted proto string of \c serialized_proto + grpc::string GetFormattedStringFromMethod( + const grpc::string& method, const grpc::string& serialized_proto, + bool is_request, bool is_json_format); + + /// Converts a binary proto string to its text or json string representation + /// for the given message type. + /// \param the serialised binary proto representation of type + /// \c message_type_name + /// \return the text- or json-formatted proto string of \c serialized_proto + grpc::string GetFormattedStringFromMessageType( const grpc::string& message_type_name, - const grpc::string& serialized_proto); + const grpc::string& serialized_proto, bool is_json_format); bool IsStreaming(const grpc::string& method, bool is_request); From aa7b8e5bc6b1ce0ea74ae2ac290ebf031b2969e2 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 16 Oct 2018 17:50:04 -0700 Subject: [PATCH 25/53] Start of channelz resolution support --- .../filters/client_channel/client_channel.cc | 43 ++++++++++++++++++- .../ext/filters/client_channel/lb_policy.h | 4 +- .../client_channel/lb_policy/grpclb/grpclb.cc | 5 ++- .../lb_policy/pick_first/pick_first.cc | 13 +++--- .../lb_policy/round_robin/round_robin.cc | 7 +-- .../client_channel/lb_policy/xds/xds.cc | 5 ++- 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 23da11363b7..0324706170a 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -518,6 +518,14 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { } // Data used to set the channel's connectivity state. bool set_connectivity_state = true; + // We only want to trace the address resolution in the follow cases: + // (a) Address resolution resulted in service config change. + // (b) Address resolution that causes number of backends to go from + // zero to non-zero. + // (c) Address resolution that causes number of backends to go from + // non-zero to zero. + // (d) Address resolution that causes a new LB policy to be created. + bool trace_this_address_resolution = false; grpc_connectivity_state connectivity_state = GRPC_CHANNEL_TRANSIENT_FAILURE; grpc_error* connectivity_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("No load balancing policy"); @@ -545,23 +553,56 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { gpr_log(GPR_INFO, "chand=%p: updating existing LB policy \"%s\" (%p)", chand, lb_policy_name.get(), chand->lb_policy.get()); } - chand->lb_policy->UpdateLocked(*chand->resolver_result); + // case (b) or (c) + trace_this_address_resolution = + chand->lb_policy->UpdateLocked(*chand->resolver_result); // No need to set the channel's connectivity state; the existing // watch on the LB policy will take care of that. set_connectivity_state = false; } else { + trace_this_address_resolution = true; // case (d) // Instantiate new LB policy. create_new_lb_policy_locked(chand, lb_policy_name.get(), &connectivity_state, &connectivity_error); + // we also log the name of the new LB policy in addition to logging this + // resolution event. + if (chand->channelz_channel != nullptr) { + char* str; + gpr_asprintf(&str, "Switched LB policy to %s", lb_policy_name.get()); + chand->channelz_channel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_copied_string(str)); + gpr_free(str); + } } // Find service config. grpc_core::UniquePtr service_config_json = get_service_config_from_resolver_result_locked(chand); + if ((service_config_json == nullptr && + chand->info_service_config_json != nullptr) || + (service_config_json != nullptr && + chand->info_service_config_json == nullptr) || + (service_config_json != nullptr && + chand->info_service_config_json != nullptr && + gpr_stricmp(service_config_json.get(), + chand->info_service_config_json.get()) != 0)) { + trace_this_address_resolution = true; // case (a) + if (chand->channelz_channel != nullptr) { + chand->channelz_channel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Service config reloaded")); + } + } // Swap out the data used by cc_get_channel_info(). gpr_mu_lock(&chand->info_mu); chand->info_lb_policy_name = std::move(lb_policy_name); chand->info_service_config_json = std::move(service_config_json); gpr_mu_unlock(&chand->info_mu); + if (trace_this_address_resolution && chand->channelz_channel != nullptr) { + chand->channelz_channel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("New addresses resolved")); + } // Clean up. grpc_channel_args_destroy(chand->resolver_result); chand->resolver_result = nullptr; diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 21f80b7b947..3940f138e66 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -95,7 +95,9 @@ class LoadBalancingPolicy /// Updates the policy with a new set of \a args from the resolver. /// Note that the LB policy gets the set of addresses from the /// GRPC_ARG_LB_ADDRESSES channel arg. - virtual void UpdateLocked(const grpc_channel_args& args) GRPC_ABSTRACT; + /// Returns true if the update caused the number of backends to go from + /// zero to non-zero, or non-zero to zero. + virtual bool UpdateLocked(const grpc_channel_args& args) GRPC_ABSTRACT; /// Finds an appropriate subchannel for a call, based on data in \a pick. /// \a pick must remain alive until the pick is complete. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 5511df7a275..4b49d10d9e0 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -123,7 +123,7 @@ class GrpcLb : public LoadBalancingPolicy { public: GrpcLb(const grpc_lb_addresses* addresses, const Args& args); - void UpdateLocked(const grpc_channel_args& args) override; + bool UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -1331,7 +1331,7 @@ void GrpcLb::ProcessChannelArgsLocked(const grpc_channel_args& args) { grpc_channel_args_destroy(lb_channel_args); } -void GrpcLb::UpdateLocked(const grpc_channel_args& args) { +bool GrpcLb::UpdateLocked(const grpc_channel_args& args) { ProcessChannelArgsLocked(args); // If fallback is configured and the RR policy already exists, update // it with the new fallback addresses. @@ -1358,6 +1358,7 @@ void GrpcLb::UpdateLocked(const grpc_channel_args& args) { &lb_channel_connectivity_, &lb_channel_on_connectivity_changed_, nullptr); } + return false; // TODO } // diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index f4dca146f71..e6f237c768d 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -46,7 +46,7 @@ class PickFirst : public LoadBalancingPolicy { public: explicit PickFirst(const Args& args); - void UpdateLocked(const grpc_channel_args& args) override; + bool UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -333,7 +333,7 @@ void PickFirst::UpdateChildRefsLocked() { child_subchannels_ = std::move(cs); } -void PickFirst::UpdateLocked(const grpc_channel_args& args) { +bool PickFirst::UpdateLocked(const grpc_channel_args& args) { AutoChildRefsUpdater guard(this); const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES); if (arg == nullptr || arg->type != GRPC_ARG_POINTER) { @@ -350,10 +350,12 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { "ignoring.", this); } - return; + return false; } const grpc_lb_addresses* addresses = static_cast(arg->value.pointer.p); + size_t old_size = + subchannel_list_ == nullptr ? 0 : subchannel_list_->num_subchannels(); if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Pick First %p received update with %" PRIuPTR " addresses", this, @@ -371,7 +373,7 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { "pf_update_empty"); subchannel_list_ = std::move(subchannel_list); // Empty list. selected_ = nullptr; - return; + return old_size != 0; } if (selected_ == nullptr) { // We don't yet have a selected subchannel, so replace the current @@ -411,7 +413,7 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { // not have contained the currently selected subchannel), drop // it, so that it doesn't override what we've done here. latest_pending_subchannel_list_.reset(); - return; + return false; } GRPC_ERROR_UNREF(error); } @@ -437,6 +439,7 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { ->CheckConnectivityStateAndStartWatchingLocked(); } } + return old_size == 0; } void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index e9ed85cf665..76d1d8b1233 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -57,7 +57,7 @@ class RoundRobin : public LoadBalancingPolicy { public: explicit RoundRobin(const Args& args); - void UpdateLocked(const grpc_channel_args& args) override; + bool UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -664,7 +664,7 @@ void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current, notify); } -void RoundRobin::UpdateLocked(const grpc_channel_args& args) { +bool RoundRobin::UpdateLocked(const grpc_channel_args& args) { const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES); AutoChildRefsUpdater guard(this); if (GPR_UNLIKELY(arg == nullptr || arg->type != GRPC_ARG_POINTER)) { @@ -677,7 +677,7 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args) { GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing update in args"), "rr_update_missing"); } - return; + return false; // TODO } grpc_lb_addresses* addresses = static_cast(arg->value.pointer.p); @@ -711,6 +711,7 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args) { // If we've started picking, start watching the new list. latest_pending_subchannel_list_->StartWatchingLocked(); } + return false; // todo } // diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index e6c94167f65..aee33b085c3 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -123,7 +123,7 @@ class XdsLb : public LoadBalancingPolicy { public: XdsLb(const grpc_lb_addresses* addresses, const Args& args); - void UpdateLocked(const grpc_channel_args& args) override; + bool UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -1325,7 +1325,7 @@ void XdsLb::ProcessChannelArgsLocked(const grpc_channel_args& args) { grpc_channel_args_destroy(lb_channel_args); } -void XdsLb::UpdateLocked(const grpc_channel_args& args) { +bool XdsLb::UpdateLocked(const grpc_channel_args& args) { ProcessChannelArgsLocked(args); // If fallback is configured and the RR policy already exists, update // it with the new fallback addresses. @@ -1352,6 +1352,7 @@ void XdsLb::UpdateLocked(const grpc_channel_args& args) { &lb_channel_connectivity_, &lb_channel_on_connectivity_changed_, nullptr); } + return false; // TODO } // From 8e76cdd2c932b5a760419ab5ba1454295be5ada8 Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 16 Oct 2018 22:41:51 -0700 Subject: [PATCH 26/53] Catch missing kick for non-polling poller --- src/core/lib/surface/completion_queue.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc index 5dc9991f703..b81ae73b4d4 100644 --- a/src/core/lib/surface/completion_queue.cc +++ b/src/core/lib/surface/completion_queue.cc @@ -79,6 +79,7 @@ typedef struct non_polling_worker { typedef struct { gpr_mu mu; + bool kicked_without_poller; non_polling_worker* root; grpc_closure* shutdown; } non_polling_poller; @@ -103,6 +104,10 @@ static grpc_error* non_polling_poller_work(grpc_pollset* pollset, grpc_millis deadline) { non_polling_poller* npp = reinterpret_cast(pollset); if (npp->shutdown) return GRPC_ERROR_NONE; + if (npp->kicked_without_poller) { + npp->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } non_polling_worker w; gpr_cv_init(&w.cv); if (worker != nullptr) *worker = reinterpret_cast(&w); @@ -148,6 +153,8 @@ static grpc_error* non_polling_poller_kick( w->kicked = true; gpr_cv_signal(&w->cv); } + } else { + p->kicked_without_poller = true; } return GRPC_ERROR_NONE; } From f7f74c5ec159904ec5f80b70b6a60d7745b9777b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 17 Oct 2018 08:52:55 +0200 Subject: [PATCH 27/53] rename memory_profile_client and memory_profile_server --- build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.yaml b/build.yaml index 31474ac90b3..b2011562a24 100644 --- a/build.yaml +++ b/build.yaml @@ -3151,7 +3151,7 @@ targets: - mac - linux - posix -- name: memory_profile_client +- name: memory_usage_client build: test run: false language: c @@ -3163,7 +3163,7 @@ targets: - gpr_test_util - gpr uses_polling: false -- name: memory_profile_server +- name: memory_usage_server build: test run: false language: c From f2db10d64c2c96565340ea1bb4312a0b4f4ff96d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 17 Oct 2018 08:53:17 +0200 Subject: [PATCH 28/53] regenerate projects --- CMakeLists.txt | 24 ++++++------- Makefile | 36 +++++++++---------- .../generated/sources_and_headers.json | 4 +-- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17327732444..e8de383ab10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -359,8 +359,8 @@ add_dependencies(buildtests_c json_stream_error_test) add_dependencies(buildtests_c json_test) add_dependencies(buildtests_c lame_client_test) add_dependencies(buildtests_c load_file_test) -add_dependencies(buildtests_c memory_profile_client) -add_dependencies(buildtests_c memory_profile_server) +add_dependencies(buildtests_c memory_usage_client) +add_dependencies(buildtests_c memory_usage_server) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c memory_usage_test) endif() @@ -8756,12 +8756,12 @@ target_link_libraries(load_file_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -add_executable(memory_profile_client +add_executable(memory_usage_client test/core/memory_usage/client.cc ) -target_include_directories(memory_profile_client +target_include_directories(memory_usage_client PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${_gRPC_SSL_INCLUDE_DIR} @@ -8774,7 +8774,7 @@ target_include_directories(memory_profile_client PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} ) -target_link_libraries(memory_profile_client +target_link_libraries(memory_usage_client ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util grpc @@ -8784,19 +8784,19 @@ target_link_libraries(memory_profile_client # avoid dependency on libstdc++ if (_gRPC_CORE_NOSTDCXX_FLAGS) - set_target_properties(memory_profile_client PROPERTIES LINKER_LANGUAGE C) - target_compile_options(memory_profile_client PRIVATE $<$:${_gRPC_CORE_NOSTDCXX_FLAGS}>) + set_target_properties(memory_usage_client PROPERTIES LINKER_LANGUAGE C) + target_compile_options(memory_usage_client PRIVATE $<$:${_gRPC_CORE_NOSTDCXX_FLAGS}>) endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -add_executable(memory_profile_server +add_executable(memory_usage_server test/core/memory_usage/server.cc ) -target_include_directories(memory_profile_server +target_include_directories(memory_usage_server PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${_gRPC_SSL_INCLUDE_DIR} @@ -8809,7 +8809,7 @@ target_include_directories(memory_profile_server PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} ) -target_link_libraries(memory_profile_server +target_link_libraries(memory_usage_server ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util grpc @@ -8819,8 +8819,8 @@ target_link_libraries(memory_profile_server # avoid dependency on libstdc++ if (_gRPC_CORE_NOSTDCXX_FLAGS) - set_target_properties(memory_profile_server PROPERTIES LINKER_LANGUAGE C) - target_compile_options(memory_profile_server PRIVATE $<$:${_gRPC_CORE_NOSTDCXX_FLAGS}>) + set_target_properties(memory_usage_server PROPERTIES LINKER_LANGUAGE C) + target_compile_options(memory_usage_server PRIVATE $<$:${_gRPC_CORE_NOSTDCXX_FLAGS}>) endif() endif (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 824b5217da8..b2f28f8a6dd 100644 --- a/Makefile +++ b/Makefile @@ -1064,8 +1064,8 @@ json_test: $(BINDIR)/$(CONFIG)/json_test lame_client_test: $(BINDIR)/$(CONFIG)/lame_client_test load_file_test: $(BINDIR)/$(CONFIG)/load_file_test low_level_ping_pong_benchmark: $(BINDIR)/$(CONFIG)/low_level_ping_pong_benchmark -memory_profile_client: $(BINDIR)/$(CONFIG)/memory_profile_client -memory_profile_server: $(BINDIR)/$(CONFIG)/memory_profile_server +memory_usage_client: $(BINDIR)/$(CONFIG)/memory_usage_client +memory_usage_server: $(BINDIR)/$(CONFIG)/memory_usage_server memory_usage_test: $(BINDIR)/$(CONFIG)/memory_usage_test message_compress_test: $(BINDIR)/$(CONFIG)/message_compress_test minimal_stack_is_minimal_test: $(BINDIR)/$(CONFIG)/minimal_stack_is_minimal_test @@ -1511,8 +1511,8 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/json_test \ $(BINDIR)/$(CONFIG)/lame_client_test \ $(BINDIR)/$(CONFIG)/load_file_test \ - $(BINDIR)/$(CONFIG)/memory_profile_client \ - $(BINDIR)/$(CONFIG)/memory_profile_server \ + $(BINDIR)/$(CONFIG)/memory_usage_client \ + $(BINDIR)/$(CONFIG)/memory_usage_server \ $(BINDIR)/$(CONFIG)/memory_usage_test \ $(BINDIR)/$(CONFIG)/message_compress_test \ $(BINDIR)/$(CONFIG)/minimal_stack_is_minimal_test \ @@ -13476,66 +13476,66 @@ endif endif -MEMORY_PROFILE_CLIENT_SRC = \ +MEMORY_USAGE_CLIENT_SRC = \ test/core/memory_usage/client.cc \ -MEMORY_PROFILE_CLIENT_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_PROFILE_CLIENT_SRC)))) +MEMORY_USAGE_CLIENT_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_USAGE_CLIENT_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/memory_profile_client: openssl_dep_error +$(BINDIR)/$(CONFIG)/memory_usage_client: openssl_dep_error else -$(BINDIR)/$(CONFIG)/memory_profile_client: $(MEMORY_PROFILE_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/memory_usage_client: $(MEMORY_USAGE_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(MEMORY_PROFILE_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/memory_profile_client + $(Q) $(LD) $(LDFLAGS) $(MEMORY_USAGE_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/memory_usage_client endif $(OBJDIR)/$(CONFIG)/test/core/memory_usage/client.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_memory_profile_client: $(MEMORY_PROFILE_CLIENT_OBJS:.o=.dep) +deps_memory_usage_client: $(MEMORY_USAGE_CLIENT_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(MEMORY_PROFILE_CLIENT_OBJS:.o=.dep) +-include $(MEMORY_USAGE_CLIENT_OBJS:.o=.dep) endif endif -MEMORY_PROFILE_SERVER_SRC = \ +MEMORY_USAGE_SERVER_SRC = \ test/core/memory_usage/server.cc \ -MEMORY_PROFILE_SERVER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_PROFILE_SERVER_SRC)))) +MEMORY_USAGE_SERVER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_USAGE_SERVER_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/memory_profile_server: openssl_dep_error +$(BINDIR)/$(CONFIG)/memory_usage_server: openssl_dep_error else -$(BINDIR)/$(CONFIG)/memory_profile_server: $(MEMORY_PROFILE_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/memory_usage_server: $(MEMORY_USAGE_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(MEMORY_PROFILE_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/memory_profile_server + $(Q) $(LD) $(LDFLAGS) $(MEMORY_USAGE_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/memory_usage_server endif $(OBJDIR)/$(CONFIG)/test/core/memory_usage/server.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_memory_profile_server: $(MEMORY_PROFILE_SERVER_OBJS:.o=.dep) +deps_memory_usage_server: $(MEMORY_USAGE_SERVER_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(MEMORY_PROFILE_SERVER_OBJS:.o=.dep) +-include $(MEMORY_USAGE_SERVER_OBJS:.o=.dep) endif endif diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 00c52baf9ea..6e1eb6e3c0f 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1593,7 +1593,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "memory_profile_client", + "name": "memory_usage_client", "src": [ "test/core/memory_usage/client.cc" ], @@ -1610,7 +1610,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "memory_profile_server", + "name": "memory_usage_server", "src": [ "test/core/memory_usage/server.cc" ], From 0e3cb55509b8f39fe79d223f32ade54e9765f3e7 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 17 Oct 2018 08:49:38 +0200 Subject: [PATCH 29/53] fix peformance_profile_* jobs on master --- tools/internal_ci/linux/grpc_performance_profile_daily.sh | 4 ++-- tools/internal_ci/linux/run_performance_profile_hourly.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh index 34d41bc04c6..0e086999238 100755 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.sh +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh @@ -24,8 +24,8 @@ CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` ./tools/run_tests/start_port_server.py || true -make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS -bins/opt/memory_profile_test +make CONFIG=opt memory_usage_test memory_usage_client memory_usage_server -j $CPUS +bins/opt/memory_usage_test bq load microbenchmarks.memory memory_usage.csv tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload || FAILED="true" diff --git a/tools/internal_ci/linux/run_performance_profile_hourly.sh b/tools/internal_ci/linux/run_performance_profile_hourly.sh index edf85c2e2c1..2be9edf756b 100755 --- a/tools/internal_ci/linux/run_performance_profile_hourly.sh +++ b/tools/internal_ci/linux/run_performance_profile_hourly.sh @@ -21,8 +21,8 @@ cd $(dirname $0)/../../.. CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` -make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS -bins/opt/memory_profile_test +make CONFIG=opt memory_usage_test memory_usage_client memory_usage_server -j $CPUS +bins/opt/memory_usage_test bq load microbenchmarks.memory memory_usage.csv tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload From 7f38ed6b24a27c7f9c31456ee4c56b1b725cf24e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 17 Oct 2018 09:05:44 +0200 Subject: [PATCH 30/53] update go interop image to go1.11 --- .../dockerfile/interoptest/grpc_interop_go/Dockerfile.template | 2 +- tools/dockerfile/interoptest/grpc_interop_go/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template index 79c55d5e56a..c908a638edc 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM golang:latest + FROM golang:1.11 <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index b136259ce90..e5e68943a4b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:latest +FROM golang:1.11 # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin From ee4cb5ce65f935b2de6e3c2a043dca3edc86cbf0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 17 Oct 2018 12:49:01 +0200 Subject: [PATCH 31/53] fix Grpc.Tools nuget layout on Linux and Mac For files without known extension (e.g. Unix binaries) , NuGet can't tell files from directories, so mention protoc and grpc_csharp_ext explicitly to avoid breaking nuget's directory layout. --- src/csharp/Grpc.Tools/Grpc.Tools.csproj | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/csharp/Grpc.Tools/Grpc.Tools.csproj b/src/csharp/Grpc.Tools/Grpc.Tools.csproj index f2cefc9826b..61fa75a4ec2 100644 --- a/src/csharp/Grpc.Tools/Grpc.Tools.csproj +++ b/src/csharp/Grpc.Tools/Grpc.Tools.csproj @@ -69,20 +69,20 @@ Linux and MacOS. Managed runtime is supplied separately in the Grpc.Core package - <_Asset PackagePath="tools/windows_x86/" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> - <_Asset PackagePath="tools/windows_x64/" Include="$(Assets_ProtoCompiler)windows_x64/protoc.exe" /> - <_Asset PackagePath="tools/linux_x86/" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> - <_Asset PackagePath="tools/linux_x64/" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> - <_Asset PackagePath="tools/macosx_x86/" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> - <_Asset PackagePath="tools/macosx_x64/" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> + <_Asset PackagePath="tools/windows_x86/protoc.exe" Include="$(Assets_ProtoCompiler)windows_x86/protoc.exe" /> + <_Asset PackagePath="tools/windows_x64/protoc.exe" Include="$(Assets_ProtoCompiler)windows_x64/protoc.exe" /> + <_Asset PackagePath="tools/linux_x86/protoc" Include="$(Assets_ProtoCompiler)linux_x86/protoc" /> + <_Asset PackagePath="tools/linux_x64/protoc" Include="$(Assets_ProtoCompiler)linux_x64/protoc" /> + <_Asset PackagePath="tools/macosx_x86/protoc" Include="$(Assets_ProtoCompiler)macos_x86/protoc" /> + <_Asset PackagePath="tools/macosx_x64/protoc" Include="$(Assets_ProtoCompiler)macos_x64/protoc" /> - <_Asset PackagePath="tools/windows_x86/" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> - <_Asset PackagePath="tools/windows_x64/" Include="$(Assets_GrpcPlugins)protoc_windows_x64/grpc_csharp_plugin.exe" /> - <_Asset PackagePath="tools/linux_x86/" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> - <_Asset PackagePath="tools/linux_x64/" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> - <_Asset PackagePath="tools/macosx_x86/" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> - <_Asset PackagePath="tools/macosx_x64/" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/windows_x86/grpc_csharp_plugin.exe" Include="$(Assets_GrpcPlugins)protoc_windows_x86/grpc_csharp_plugin.exe" /> + <_Asset PackagePath="tools/windows_x64/grpc_csharp_plugin.exe" Include="$(Assets_GrpcPlugins)protoc_windows_x64/grpc_csharp_plugin.exe" /> + <_Asset PackagePath="tools/linux_x86/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_linux_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/linux_x64/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_linux_x64/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/macosx_x86/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_macos_x86/grpc_csharp_plugin" /> + <_Asset PackagePath="tools/macosx_x64/grpc_csharp_plugin" Include="$(Assets_GrpcPlugins)protoc_macos_x64/grpc_csharp_plugin" /> From 231b8639220dc6204ad153469091fff56a730481 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 17 Oct 2018 09:29:05 -0700 Subject: [PATCH 32/53] address clang-tidy problems --- src/cpp/server/health/default_health_check_service.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index d7e1356bde2..0c03fdf17a8 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -72,7 +72,8 @@ void DefaultHealthCheckService::RegisterCallHandler( std::unique_lock lock(mu_); ServiceData& service_data = services_map_[service_name]; service_data.AddCallHandler(handler /* copies ref */); - handler->SendHealth(std::move(handler), service_data.GetServingStatus()); + HealthCheckServiceImpl::CallHandler* h = handler.get(); + h->SendHealth(std::move(handler), service_data.GetServingStatus()); } void DefaultHealthCheckService::UnregisterCallHandler( @@ -115,7 +116,7 @@ void DefaultHealthCheckService::ServiceData::AddCallHandler( void DefaultHealthCheckService::ServiceData::RemoveCallHandler( std::shared_ptr handler) { - call_handlers_.erase(std::move(handler)); + call_handlers_.erase(handler); } // From 1a9c197987b5ecc49fd0e157da42c8c0c715a631 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 17 Oct 2018 10:16:30 -0700 Subject: [PATCH 33/53] clang-tidy fixes --- src/core/ext/filters/client_channel/lb_policy/xds/xds.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index e6c94167f65..16e910b2d9b 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -673,7 +673,7 @@ bool XdsLb::BalancerCallState::LoadReportCountersAreZero( request->client_stats.num_calls_finished_with_client_failed_to_send == 0 && request->client_stats.num_calls_finished_known_received == 0 && - (drop_entries == nullptr || drop_entries->size() == 0); + (drop_entries == nullptr || drop_entries->empty()); } void XdsLb::BalancerCallState::SendClientLoadReportLocked() { @@ -1275,9 +1275,9 @@ grpc_connectivity_state XdsLb::CheckConnectivityLocked( } void XdsLb::NotifyOnStateChangeLocked(grpc_connectivity_state* current, - grpc_closure* notify) { + grpc_closure* closure) { grpc_connectivity_state_notify_on_state_change(&state_tracker_, current, - notify); + closure); } void XdsLb::ProcessChannelArgsLocked(const grpc_channel_args& args) { From 265eace8e6375bfd0ec88562e7988ea6de859349 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 17 Oct 2018 11:59:43 -0700 Subject: [PATCH 34/53] reviewer feedback --- .../filters/client_channel/client_channel.cc | 58 ++++++++++--------- .../ext/filters/client_channel/lb_policy.h | 4 +- .../client_channel/lb_policy/grpclb/grpclb.cc | 5 +- .../lb_policy/pick_first/pick_first.cc | 13 ++--- .../lb_policy/round_robin/round_robin.cc | 7 +-- .../client_channel/lb_policy/xds/xds.cc | 5 +- 6 files changed, 44 insertions(+), 48 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 0324706170a..b3ff8653d4f 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -525,7 +525,9 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { // (c) Address resolution that causes number of backends to go from // non-zero to zero. // (d) Address resolution that causes a new LB policy to be created. - bool trace_this_address_resolution = false; + // + // we track a list of strings to eventually be concatenated and traced. + grpc_core::InlinedVector trace_strings; grpc_connectivity_state connectivity_state = GRPC_CHANNEL_TRANSIENT_FAILURE; grpc_error* connectivity_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("No load balancing policy"); @@ -553,44 +555,51 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { gpr_log(GPR_INFO, "chand=%p: updating existing LB policy \"%s\" (%p)", chand, lb_policy_name.get(), chand->lb_policy.get()); } - // case (b) or (c) - trace_this_address_resolution = - chand->lb_policy->UpdateLocked(*chand->resolver_result); + chand->lb_policy->UpdateLocked(*chand->resolver_result); // No need to set the channel's connectivity state; the existing // watch on the LB policy will take care of that. set_connectivity_state = false; } else { - trace_this_address_resolution = true; // case (d) // Instantiate new LB policy. create_new_lb_policy_locked(chand, lb_policy_name.get(), &connectivity_state, &connectivity_error); - // we also log the name of the new LB policy in addition to logging this - // resolution event. if (chand->channelz_channel != nullptr) { char* str; gpr_asprintf(&str, "Switched LB policy to %s", lb_policy_name.get()); - chand->channelz_channel->AddTraceEvent( - grpc_core::channelz::ChannelTrace::Severity::Info, - grpc_slice_from_copied_string(str)); - gpr_free(str); + trace_strings.push_back(str); } } // Find service config. grpc_core::UniquePtr service_config_json = get_service_config_from_resolver_result_locked(chand); - if ((service_config_json == nullptr && - chand->info_service_config_json != nullptr) || - (service_config_json != nullptr && - chand->info_service_config_json == nullptr) || - (service_config_json != nullptr && - chand->info_service_config_json != nullptr && - gpr_stricmp(service_config_json.get(), - chand->info_service_config_json.get()) != 0)) { - trace_this_address_resolution = true; // case (a) - if (chand->channelz_channel != nullptr) { + // Note: It's safe to use chand->info_lb_policy_name here without + // taking a lock on chand->info_mu, because this function is the + // only thing that modifies its value, and it can only be invoked + // once at any given time. + if (chand->channelz_channel != nullptr) { + if ((service_config_json == nullptr && + chand->info_service_config_json != nullptr) || + (service_config_json != nullptr && + chand->info_service_config_json == nullptr) || + (service_config_json != nullptr && + chand->info_service_config_json != nullptr && + strcmp(service_config_json.get(), + chand->info_service_config_json.get()) != 0)) { + trace_strings.push_back(gpr_strdup("Service config reloaded")); + } + //TODO + // const grpc_arg* channel_arg = + // grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_ADDRESSES); + // if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_POINTER) { + // grpc_lb_addresses* addresses = + // static_cast(channel_arg->value.pointer.p); + // } else { + // } + if (!trace_strings.empty()) { + // TODO, concatenate the strings from trace_strings. chand->channelz_channel->AddTraceEvent( grpc_core::channelz::ChannelTrace::Severity::Info, - grpc_slice_from_static_string("Service config reloaded")); + grpc_slice_from_static_string("New addresses resolved")); } } // Swap out the data used by cc_get_channel_info(). @@ -598,11 +607,6 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { chand->info_lb_policy_name = std::move(lb_policy_name); chand->info_service_config_json = std::move(service_config_json); gpr_mu_unlock(&chand->info_mu); - if (trace_this_address_resolution && chand->channelz_channel != nullptr) { - chand->channelz_channel->AddTraceEvent( - grpc_core::channelz::ChannelTrace::Severity::Info, - grpc_slice_from_static_string("New addresses resolved")); - } // Clean up. grpc_channel_args_destroy(chand->resolver_result); chand->resolver_result = nullptr; diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 3940f138e66..21f80b7b947 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -95,9 +95,7 @@ class LoadBalancingPolicy /// Updates the policy with a new set of \a args from the resolver. /// Note that the LB policy gets the set of addresses from the /// GRPC_ARG_LB_ADDRESSES channel arg. - /// Returns true if the update caused the number of backends to go from - /// zero to non-zero, or non-zero to zero. - virtual bool UpdateLocked(const grpc_channel_args& args) GRPC_ABSTRACT; + virtual void UpdateLocked(const grpc_channel_args& args) GRPC_ABSTRACT; /// Finds an appropriate subchannel for a call, based on data in \a pick. /// \a pick must remain alive until the pick is complete. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 4b49d10d9e0..5511df7a275 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -123,7 +123,7 @@ class GrpcLb : public LoadBalancingPolicy { public: GrpcLb(const grpc_lb_addresses* addresses, const Args& args); - bool UpdateLocked(const grpc_channel_args& args) override; + void UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -1331,7 +1331,7 @@ void GrpcLb::ProcessChannelArgsLocked(const grpc_channel_args& args) { grpc_channel_args_destroy(lb_channel_args); } -bool GrpcLb::UpdateLocked(const grpc_channel_args& args) { +void GrpcLb::UpdateLocked(const grpc_channel_args& args) { ProcessChannelArgsLocked(args); // If fallback is configured and the RR policy already exists, update // it with the new fallback addresses. @@ -1358,7 +1358,6 @@ bool GrpcLb::UpdateLocked(const grpc_channel_args& args) { &lb_channel_connectivity_, &lb_channel_on_connectivity_changed_, nullptr); } - return false; // TODO } // diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index e6f237c768d..f4dca146f71 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -46,7 +46,7 @@ class PickFirst : public LoadBalancingPolicy { public: explicit PickFirst(const Args& args); - bool UpdateLocked(const grpc_channel_args& args) override; + void UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -333,7 +333,7 @@ void PickFirst::UpdateChildRefsLocked() { child_subchannels_ = std::move(cs); } -bool PickFirst::UpdateLocked(const grpc_channel_args& args) { +void PickFirst::UpdateLocked(const grpc_channel_args& args) { AutoChildRefsUpdater guard(this); const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES); if (arg == nullptr || arg->type != GRPC_ARG_POINTER) { @@ -350,12 +350,10 @@ bool PickFirst::UpdateLocked(const grpc_channel_args& args) { "ignoring.", this); } - return false; + return; } const grpc_lb_addresses* addresses = static_cast(arg->value.pointer.p); - size_t old_size = - subchannel_list_ == nullptr ? 0 : subchannel_list_->num_subchannels(); if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Pick First %p received update with %" PRIuPTR " addresses", this, @@ -373,7 +371,7 @@ bool PickFirst::UpdateLocked(const grpc_channel_args& args) { "pf_update_empty"); subchannel_list_ = std::move(subchannel_list); // Empty list. selected_ = nullptr; - return old_size != 0; + return; } if (selected_ == nullptr) { // We don't yet have a selected subchannel, so replace the current @@ -413,7 +411,7 @@ bool PickFirst::UpdateLocked(const grpc_channel_args& args) { // not have contained the currently selected subchannel), drop // it, so that it doesn't override what we've done here. latest_pending_subchannel_list_.reset(); - return false; + return; } GRPC_ERROR_UNREF(error); } @@ -439,7 +437,6 @@ bool PickFirst::UpdateLocked(const grpc_channel_args& args) { ->CheckConnectivityStateAndStartWatchingLocked(); } } - return old_size == 0; } void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 76d1d8b1233..e9ed85cf665 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -57,7 +57,7 @@ class RoundRobin : public LoadBalancingPolicy { public: explicit RoundRobin(const Args& args); - bool UpdateLocked(const grpc_channel_args& args) override; + void UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -664,7 +664,7 @@ void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current, notify); } -bool RoundRobin::UpdateLocked(const grpc_channel_args& args) { +void RoundRobin::UpdateLocked(const grpc_channel_args& args) { const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES); AutoChildRefsUpdater guard(this); if (GPR_UNLIKELY(arg == nullptr || arg->type != GRPC_ARG_POINTER)) { @@ -677,7 +677,7 @@ bool RoundRobin::UpdateLocked(const grpc_channel_args& args) { GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing update in args"), "rr_update_missing"); } - return false; // TODO + return; } grpc_lb_addresses* addresses = static_cast(arg->value.pointer.p); @@ -711,7 +711,6 @@ bool RoundRobin::UpdateLocked(const grpc_channel_args& args) { // If we've started picking, start watching the new list. latest_pending_subchannel_list_->StartWatchingLocked(); } - return false; // todo } // diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index aee33b085c3..e6c94167f65 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -123,7 +123,7 @@ class XdsLb : public LoadBalancingPolicy { public: XdsLb(const grpc_lb_addresses* addresses, const Args& args); - bool UpdateLocked(const grpc_channel_args& args) override; + void UpdateLocked(const grpc_channel_args& args) override; bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, @@ -1325,7 +1325,7 @@ void XdsLb::ProcessChannelArgsLocked(const grpc_channel_args& args) { grpc_channel_args_destroy(lb_channel_args); } -bool XdsLb::UpdateLocked(const grpc_channel_args& args) { +void XdsLb::UpdateLocked(const grpc_channel_args& args) { ProcessChannelArgsLocked(args); // If fallback is configured and the RR policy already exists, update // it with the new fallback addresses. @@ -1352,7 +1352,6 @@ bool XdsLb::UpdateLocked(const grpc_channel_args& args) { &lb_channel_connectivity_, &lb_channel_on_connectivity_changed_, nullptr); } - return false; // TODO } // From cc5fe63da0785a4f29346ac43e529d924c6f338d Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Wed, 17 Oct 2018 16:27:59 -0400 Subject: [PATCH 35/53] Fix bazel for //test/cpp/microbenchmarks/... Fix a build failure due to missed dependency and add the bm_create_call benchmark to the build targets. --- test/cpp/microbenchmarks/BUILD | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 0c3b9ef8167..6c1859ff4fe 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -24,7 +24,7 @@ grpc_cc_test( external_deps = [ "benchmark", ], - deps = ["//test/core/util:gpr_test_util",] + deps = ["//test/core/util:gpr_test_util"], ) grpc_cc_library( @@ -41,6 +41,7 @@ grpc_cc_library( ], deps = [ "//:grpc++_unsecure", + "//:lb_server_load_reporting_filter", "//src/proto/grpc/testing:echo_proto", "//test/core/util:grpc_test_util_unsecure", "//test/cpp/util:test_config", @@ -68,6 +69,13 @@ grpc_cc_binary( deps = [":helpers"], ) +grpc_cc_binary( + name = "bm_call_create", + testonly = 1, + srcs = ["bm_call_create.cc"], + deps = [":helpers"], +) + grpc_cc_binary( name = "bm_cq", testonly = 1, From 03dbb8c1e2ee6261e1b8802bb7dd0be9a0b8bd46 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 17 Oct 2018 14:16:49 -0700 Subject: [PATCH 36/53] reviewer feedback --- .../filters/client_channel/client_channel.cc | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index b3ff8653d4f..95d1f410945 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -131,6 +131,8 @@ typedef struct client_channel_channel_data { grpc_core::UniquePtr info_service_config_json; /* backpointer to grpc_channel's channelz node */ grpc_core::channelz::ClientChannelNode* channelz_channel; + /* caches if the last resolution event led to zero addresses */ + bool previous_resolution_zero_num_addresses; } channel_data; typedef struct { @@ -572,7 +574,7 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { // Find service config. grpc_core::UniquePtr service_config_json = get_service_config_from_resolver_result_locked(chand); - // Note: It's safe to use chand->info_lb_policy_name here without + // Note: It's safe to use chand->info_service_config_json here without // taking a lock on chand->info_mu, because this function is the // only thing that modifies its value, and it can only be invoked // once at any given time. @@ -585,21 +587,45 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { chand->info_service_config_json != nullptr && strcmp(service_config_json.get(), chand->info_service_config_json.get()) != 0)) { + // TODO(ncteisen): might be worth somehow including a snippet of the + // config in the trace, at the risk of bloating the trace logs. trace_strings.push_back(gpr_strdup("Service config reloaded")); } - //TODO - // const grpc_arg* channel_arg = - // grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_ADDRESSES); - // if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_POINTER) { - // grpc_lb_addresses* addresses = - // static_cast(channel_arg->value.pointer.p); - // } else { - // } + int zero_num_addresses = true; + const grpc_arg* channel_arg = + grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_ADDRESSES); + if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_POINTER) { + grpc_lb_addresses* addresses = + static_cast(channel_arg->value.pointer.p); + if (addresses->num_addresses > 0) { + zero_num_addresses = false; + } + } + if (zero_num_addresses && + !chand->previous_resolution_zero_num_addresses) { + trace_strings.push_back(gpr_strdup("Address list became empty")); + } else if (!zero_num_addresses && + chand->previous_resolution_zero_num_addresses) { + trace_strings.push_back(gpr_strdup("Address list became non-empty")); + } + chand->previous_resolution_zero_num_addresses = zero_num_addresses; if (!trace_strings.empty()) { - // TODO, concatenate the strings from trace_strings. + gpr_strvec v; + gpr_strvec_init(&v); + gpr_strvec_add(&v, gpr_strdup("Resolution event: ")); + bool is_first = 1; + for (size_t i = 0; i < trace_strings.size(); ++i) { + if (!is_first) gpr_strvec_add(&v, gpr_strdup(", ")); + is_first = false; + gpr_strvec_add(&v, trace_strings[i]); + } + char* flat; + size_t flat_len = 0; + flat = gpr_strvec_flatten(&v, &flat_len); chand->channelz_channel->AddTraceEvent( grpc_core::channelz::ChannelTrace::Severity::Info, - grpc_slice_from_static_string("New addresses resolved")); + grpc_slice_new(flat, flat_len, gpr_free)); + gpr_strvec_destroy(&v); } } // Swap out the data used by cc_get_channel_info(). @@ -770,6 +796,7 @@ static grpc_error* cc_init_channel_elem(grpc_channel_element* elem, arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_ENABLE_RETRIES); chand->enable_retries = grpc_channel_arg_get_bool(arg, true); chand->channelz_channel = nullptr; + chand->previous_resolution_zero_num_addresses = true; // Record client channel factory. arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_CLIENT_CHANNEL_FACTORY); From f6e84176270df6012a74686a651a0a397e05ec51 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 17 Oct 2018 19:44:52 -0700 Subject: [PATCH 37/53] Move package_options.proto file for ruby test --- .../spec/pb/codegen}/grpc/testing/package_options.proto | 0 src/ruby/spec/pb/codegen/package_option_spec.rb | 5 ++--- 2 files changed, 2 insertions(+), 3 deletions(-) rename src/{proto => ruby/spec/pb/codegen}/grpc/testing/package_options.proto (100%) diff --git a/src/proto/grpc/testing/package_options.proto b/src/ruby/spec/pb/codegen/grpc/testing/package_options.proto similarity index 100% rename from src/proto/grpc/testing/package_options.proto rename to src/ruby/spec/pb/codegen/grpc/testing/package_options.proto diff --git a/src/ruby/spec/pb/codegen/package_option_spec.rb b/src/ruby/spec/pb/codegen/package_option_spec.rb index 46d23cd6518..0ebd503d79d 100644 --- a/src/ruby/spec/pb/codegen/package_option_spec.rb +++ b/src/ruby/spec/pb/codegen/package_option_spec.rb @@ -21,9 +21,8 @@ describe 'Code Generation Options' do fail 'CONFIG env variable unexpectedly unset' unless ENV['CONFIG'] bins_sub_dir = ENV['CONFIG'] - src_dir = File.join(File.dirname(__FILE__), '..', '..', '..', '..') - pb_dir = File.join(src_dir, 'proto') - bins_dir = File.join(src_dir, '..', 'bins', bins_sub_dir) + pb_dir = File.dirname(__FILE__) + bins_dir = File.join('..', '..', '..', '..', '..', 'bins', bins_sub_dir) plugin = File.join(bins_dir, 'grpc_ruby_plugin') protoc = File.join(bins_dir, 'protobuf', 'protoc') From 30cc76e1f7fa647bd3ea5146b8ac3518b816ebb5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 18 Oct 2018 13:58:50 +0200 Subject: [PATCH 38/53] fix memory_usage_test --- test/core/memory_usage/memory_usage_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/memory_usage/memory_usage_test.cc b/test/core/memory_usage/memory_usage_test.cc index c170f5ad265..5c35b4e1d36 100644 --- a/test/core/memory_usage/memory_usage_test.cc +++ b/test/core/memory_usage/memory_usage_test.cc @@ -43,7 +43,7 @@ int main(int argc, char** argv) { strcpy(root, "."); } /* start the server */ - gpr_asprintf(&args[0], "%s/memory_profile_server%s", root, + gpr_asprintf(&args[0], "%s/memory_usage_server%s", root, gpr_subprocess_binary_extension()); args[1] = const_cast("--bind"); gpr_join_host_port(&args[2], "::", port); @@ -53,7 +53,7 @@ int main(int argc, char** argv) { gpr_free(args[2]); /* start the client */ - gpr_asprintf(&args[0], "%s/memory_profile_client%s", root, + gpr_asprintf(&args[0], "%s/memory_usage_client%s", root, gpr_subprocess_binary_extension()); args[1] = const_cast("--target"); gpr_join_host_port(&args[2], "127.0.0.1", port); From a60226726ae6076916db0e1d616338a1a211770d Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 18 Oct 2018 10:49:25 -0700 Subject: [PATCH 39/53] reviewer feedback --- .../filters/client_channel/client_channel.cc | 122 ++++++++++-------- test/core/end2end/tests/channelz.cc | 2 + test/core/end2end/tests/retry_streaming.cc | 55 +++++--- 3 files changed, 107 insertions(+), 72 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 95d1f410945..64e206ec631 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -131,8 +131,8 @@ typedef struct client_channel_channel_data { grpc_core::UniquePtr info_service_config_json; /* backpointer to grpc_channel's channelz node */ grpc_core::channelz::ClientChannelNode* channelz_channel; - /* caches if the last resolution event led to zero addresses */ - bool previous_resolution_zero_num_addresses; + /* caches if the last resolution event contained addresses */ + bool previous_resolution_contained_addresses; } channel_data; typedef struct { @@ -403,6 +403,8 @@ static void request_reresolution_locked(void* arg, grpc_error* error) { chand->lb_policy->SetReresolutionClosureLocked(&args->closure); } +using TraceStringVector = grpc_core::InlinedVector; + // Creates a new LB policy, replacing any previous one. // If the new policy is created successfully, sets *connectivity_state and // *connectivity_error to its initial connectivity state; otherwise, @@ -410,7 +412,7 @@ static void request_reresolution_locked(void* arg, grpc_error* error) { static void create_new_lb_policy_locked( channel_data* chand, char* lb_policy_name, grpc_connectivity_state* connectivity_state, - grpc_error** connectivity_error) { + grpc_error** connectivity_error, TraceStringVector* trace_strings) { grpc_core::LoadBalancingPolicy::Args lb_policy_args; lb_policy_args.combiner = chand->combiner; lb_policy_args.client_channel_factory = chand->client_channel_factory; @@ -420,11 +422,21 @@ static void create_new_lb_policy_locked( lb_policy_name, lb_policy_args); if (GPR_UNLIKELY(new_lb_policy == nullptr)) { gpr_log(GPR_ERROR, "could not create LB policy \"%s\"", lb_policy_name); + if (chand->channelz_channel != nullptr) { + char* str; + gpr_asprintf(&str, "Could not create LB policy \'%s\'", lb_policy_name); + trace_strings->push_back(str); + } } else { if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p: created new LB policy \"%s\" (%p)", chand, lb_policy_name, new_lb_policy.get()); } + if (chand->channelz_channel != nullptr) { + char* str; + gpr_asprintf(&str, "Created new LB policy \'%s\'", lb_policy_name); + trace_strings->push_back(str); + } // Swap out the LB policy and update the fds in // chand->interested_parties. if (chand->lb_policy != nullptr) { @@ -499,6 +511,51 @@ get_service_config_from_resolver_result_locked(channel_data* chand) { return grpc_core::UniquePtr(gpr_strdup(service_config_json)); } +static void check_for_important_resolution_change( + channel_data* chand, TraceStringVector* trace_strings) { + int resolution_contains_addresses = false; + const grpc_arg* channel_arg = + grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_ADDRESSES); + if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_POINTER) { + grpc_lb_addresses* addresses = + static_cast(channel_arg->value.pointer.p); + if (addresses->num_addresses > 0) { + resolution_contains_addresses = true; + } + } + if (!resolution_contains_addresses && + chand->previous_resolution_contained_addresses) { + trace_strings->push_back(gpr_strdup("Address list became empty")); + } else if (resolution_contains_addresses && + !chand->previous_resolution_contained_addresses) { + trace_strings->push_back(gpr_strdup("Address list became non-empty")); + } + chand->previous_resolution_contained_addresses = + resolution_contains_addresses; +} + +static void concatenate_and_add_channel_trace( + channel_data* chand, TraceStringVector* trace_strings) { + if (!trace_strings->empty()) { + gpr_strvec v; + gpr_strvec_init(&v); + gpr_strvec_add(&v, gpr_strdup("Resolution event: ")); + bool is_first = 1; + for (size_t i = 0; i < trace_strings->size(); ++i) { + if (!is_first) gpr_strvec_add(&v, gpr_strdup(", ")); + is_first = false; + gpr_strvec_add(&v, (*trace_strings)[i]); + } + char* flat; + size_t flat_len = 0; + flat = gpr_strvec_flatten(&v, &flat_len); + chand->channelz_channel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_new(flat, flat_len, gpr_free)); + gpr_strvec_destroy(&v); + } +} + // Callback invoked when a resolver result is available. static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { channel_data* chand = static_cast(arg); @@ -529,7 +586,7 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { // (d) Address resolution that causes a new LB policy to be created. // // we track a list of strings to eventually be concatenated and traced. - grpc_core::InlinedVector trace_strings; + TraceStringVector trace_strings; grpc_connectivity_state connectivity_state = GRPC_CHANNEL_TRANSIENT_FAILURE; grpc_error* connectivity_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("No load balancing policy"); @@ -564,12 +621,8 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { } else { // Instantiate new LB policy. create_new_lb_policy_locked(chand, lb_policy_name.get(), - &connectivity_state, &connectivity_error); - if (chand->channelz_channel != nullptr) { - char* str; - gpr_asprintf(&str, "Switched LB policy to %s", lb_policy_name.get()); - trace_strings.push_back(str); - } + &connectivity_state, &connectivity_error, + &trace_strings); } // Find service config. grpc_core::UniquePtr service_config_json = @@ -579,54 +632,17 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { // only thing that modifies its value, and it can only be invoked // once at any given time. if (chand->channelz_channel != nullptr) { - if ((service_config_json == nullptr && - chand->info_service_config_json != nullptr) || - (service_config_json != nullptr && - chand->info_service_config_json == nullptr) || + if (((service_config_json == nullptr) != + (chand->info_service_config_json == nullptr)) || (service_config_json != nullptr && - chand->info_service_config_json != nullptr && strcmp(service_config_json.get(), chand->info_service_config_json.get()) != 0)) { // TODO(ncteisen): might be worth somehow including a snippet of the // config in the trace, at the risk of bloating the trace logs. - trace_strings.push_back(gpr_strdup("Service config reloaded")); - } - int zero_num_addresses = true; - const grpc_arg* channel_arg = - grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_ADDRESSES); - if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_POINTER) { - grpc_lb_addresses* addresses = - static_cast(channel_arg->value.pointer.p); - if (addresses->num_addresses > 0) { - zero_num_addresses = false; - } - } - if (zero_num_addresses && - !chand->previous_resolution_zero_num_addresses) { - trace_strings.push_back(gpr_strdup("Address list became empty")); - } else if (!zero_num_addresses && - chand->previous_resolution_zero_num_addresses) { - trace_strings.push_back(gpr_strdup("Address list became non-empty")); - } - chand->previous_resolution_zero_num_addresses = zero_num_addresses; - if (!trace_strings.empty()) { - gpr_strvec v; - gpr_strvec_init(&v); - gpr_strvec_add(&v, gpr_strdup("Resolution event: ")); - bool is_first = 1; - for (size_t i = 0; i < trace_strings.size(); ++i) { - if (!is_first) gpr_strvec_add(&v, gpr_strdup(", ")); - is_first = false; - gpr_strvec_add(&v, trace_strings[i]); - } - char* flat; - size_t flat_len = 0; - flat = gpr_strvec_flatten(&v, &flat_len); - chand->channelz_channel->AddTraceEvent( - grpc_core::channelz::ChannelTrace::Severity::Info, - grpc_slice_new(flat, flat_len, gpr_free)); - gpr_strvec_destroy(&v); + trace_strings.push_back(gpr_strdup("Service config changed")); } + check_for_important_resolution_change(chand, &trace_strings); + concatenate_and_add_channel_trace(chand, &trace_strings); } // Swap out the data used by cc_get_channel_info(). gpr_mu_lock(&chand->info_mu); @@ -796,7 +812,7 @@ static grpc_error* cc_init_channel_elem(grpc_channel_element* elem, arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_ENABLE_RETRIES); chand->enable_retries = grpc_channel_arg_get_bool(arg, true); chand->channelz_channel = nullptr; - chand->previous_resolution_zero_num_addresses = true; + chand->previous_resolution_contained_addresses = false; // Record client channel factory. arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_CLIENT_CHANNEL_FACTORY); diff --git a/test/core/end2end/tests/channelz.cc b/test/core/end2end/tests/channelz.cc index a812994435e..7c61b7910be 100644 --- a/test/core/end2end/tests/channelz.cc +++ b/test/core/end2end/tests/channelz.cc @@ -288,6 +288,8 @@ static void test_channelz_with_channel_trace(grpc_end2end_test_config config) { grpc_server_get_channelz_node(f.server); GPR_ASSERT(channelz_server != nullptr); + run_one_request(config, f, true); + char* json = channelz_channel->RenderJsonString(); GPR_ASSERT(json != nullptr); gpr_log(GPR_INFO, "%s", json); diff --git a/test/core/end2end/tests/retry_streaming.cc b/test/core/end2end/tests/retry_streaming.cc index d06d124ca42..d52574fc589 100644 --- a/test/core/end2end/tests/retry_streaming.cc +++ b/test/core/end2end/tests/retry_streaming.cc @@ -28,6 +28,9 @@ #include #include +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/server.h" + #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" @@ -133,25 +136,30 @@ static void test_retry_streaming(grpc_end2end_test_config config) { int was_cancelled = 2; char* peer; - grpc_arg arg; - arg.type = GRPC_ARG_STRING; - arg.key = const_cast(GRPC_ARG_SERVICE_CONFIG); - arg.value.string = const_cast( - "{\n" - " \"methodConfig\": [ {\n" - " \"name\": [\n" - " { \"service\": \"service\", \"method\": \"method\" }\n" - " ],\n" - " \"retryPolicy\": {\n" - " \"maxAttempts\": 3,\n" - " \"initialBackoff\": \"1s\",\n" - " \"maxBackoff\": \"120s\",\n" - " \"backoffMultiplier\": 1.6,\n" - " \"retryableStatusCodes\": [ \"ABORTED\" ]\n" - " }\n" - " } ]\n" - "}"); - grpc_channel_args client_args = {1, &arg}; + grpc_arg arg[] = { + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), + 1024 * 8), + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_ENABLE_CHANNELZ), true), + grpc_channel_arg_string_create( + const_cast(GRPC_ARG_SERVICE_CONFIG), + const_cast( + "{\n" + " \"methodConfig\": [ {\n" + " \"name\": [\n" + " { \"service\": \"service\", \"method\": \"method\" }\n" + " ],\n" + " \"retryPolicy\": {\n" + " \"maxAttempts\": 3,\n" + " \"initialBackoff\": \"1s\",\n" + " \"maxBackoff\": \"120s\",\n" + " \"backoffMultiplier\": 1.6,\n" + " \"retryableStatusCodes\": [ \"ABORTED\" ]\n" + " }\n" + " } ]\n" + "}"))}; + grpc_channel_args client_args = {GPR_ARRAY_SIZE(arg), arg}; grpc_end2end_test_fixture f = begin_test(config, "retry_streaming", &client_args, nullptr); @@ -161,6 +169,9 @@ static void test_retry_streaming(grpc_end2end_test_config config) { c = grpc_channel_create_call(f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/service/method"), nullptr, deadline, nullptr); + grpc_core::channelz::ChannelNode* channelz_channel = + grpc_channel_get_channelz_node(f.client); + GPR_ASSERT(c); peer = grpc_call_get_peer(c); @@ -384,6 +395,11 @@ static void test_retry_streaming(grpc_end2end_test_config config) { GPR_ASSERT(0 == call_details.flags); GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(channelz_channel != nullptr); + char* json = channelz_channel->RenderJsonString(); + GPR_ASSERT(json != nullptr); + gpr_log(GPR_INFO, "%s", json); + grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); grpc_metadata_array_destroy(&trailing_metadata_recv); @@ -414,6 +430,7 @@ static void test_retry_streaming(grpc_end2end_test_config config) { void retry_streaming(grpc_end2end_test_config config) { GPR_ASSERT(config.feature_mask & FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL); + test_retry_streaming(config); } From 0582a003d5385ce16ec13d5e5d1128913f9b59c6 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 18 Oct 2018 11:42:32 -0700 Subject: [PATCH 40/53] reviewer comments --- .../ext/filters/client_channel/client_channel.cc | 8 ++++---- test/core/end2end/tests/retry_streaming.cc | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 64e206ec631..daf1b89b094 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -511,7 +511,7 @@ get_service_config_from_resolver_result_locked(channel_data* chand) { return grpc_core::UniquePtr(gpr_strdup(service_config_json)); } -static void check_for_important_resolution_change( +static void maybe_add_trace_message_for_address_changes_locked( channel_data* chand, TraceStringVector* trace_strings) { int resolution_contains_addresses = false; const grpc_arg* channel_arg = @@ -534,7 +534,7 @@ static void check_for_important_resolution_change( resolution_contains_addresses; } -static void concatenate_and_add_channel_trace( +static void concatenate_and_add_channel_trace_locked( channel_data* chand, TraceStringVector* trace_strings) { if (!trace_strings->empty()) { gpr_strvec v; @@ -641,8 +641,8 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { // config in the trace, at the risk of bloating the trace logs. trace_strings.push_back(gpr_strdup("Service config changed")); } - check_for_important_resolution_change(chand, &trace_strings); - concatenate_and_add_channel_trace(chand, &trace_strings); + maybe_add_trace_message_for_address_changes_locked(chand, &trace_strings); + concatenate_and_add_channel_trace_locked(chand, &trace_strings); } // Swap out the data used by cc_get_channel_info(). gpr_mu_lock(&chand->info_mu); diff --git a/test/core/end2end/tests/retry_streaming.cc b/test/core/end2end/tests/retry_streaming.cc index d52574fc589..94a27faf7b2 100644 --- a/test/core/end2end/tests/retry_streaming.cc +++ b/test/core/end2end/tests/retry_streaming.cc @@ -136,7 +136,7 @@ static void test_retry_streaming(grpc_end2end_test_config config) { int was_cancelled = 2; char* peer; - grpc_arg arg[] = { + grpc_arg args[] = { grpc_channel_arg_integer_create( const_cast(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), 1024 * 8), @@ -159,7 +159,7 @@ static void test_retry_streaming(grpc_end2end_test_config config) { " }\n" " } ]\n" "}"))}; - grpc_channel_args client_args = {GPR_ARRAY_SIZE(arg), arg}; + grpc_channel_args client_args = {GPR_ARRAY_SIZE(args), args}; grpc_end2end_test_fixture f = begin_test(config, "retry_streaming", &client_args, nullptr); @@ -399,6 +399,15 @@ static void test_retry_streaming(grpc_end2end_test_config config) { char* json = channelz_channel->RenderJsonString(); GPR_ASSERT(json != nullptr); gpr_log(GPR_INFO, "%s", json); + GPR_ASSERT(nullptr != strstr(json, "\"trace\"")); + GPR_ASSERT(nullptr != strstr(json, "\"description\":\"Channel created\"")); + GPR_ASSERT(nullptr != strstr(json, "\"severity\":\"CT_INFO\"")); + GPR_ASSERT(nullptr != strstr(json, "Resolution event")); + GPR_ASSERT(nullptr != strstr(json, "Created new LB policy")); + GPR_ASSERT(nullptr != strstr(json, "Service config changed")); + GPR_ASSERT(nullptr != strstr(json, "Address list became non-empty")); + GPR_ASSERT(nullptr != strstr(json, "Channel state change to CONNECTING")); + gpr_free(json); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); From a33e796575166c8ba69e6317e26fdbf79c03cfb0 Mon Sep 17 00:00:00 2001 From: Vishal Powar Date: Tue, 16 Oct 2018 21:56:22 -0700 Subject: [PATCH 41/53] Changes to register xds plugin init and shutdown. Also, - Changes to extract grpclb_proto into its own build target - Remove client_load_reporting_filter from xds plugin. --- BUILD | 61 ++++---- CMakeLists.txt | 16 +- Makefile | 17 ++- build.yaml | 54 +++---- config.m4 | 7 +- config.w32 | 7 +- gRPC-C++.podspec | 6 +- gRPC-Core.podspec | 18 ++- grpc.gemspec | 12 +- grpc.gyp | 16 +- package.xml | 12 +- .../xds/client_load_reporting_filter.cc | 140 ------------------ .../xds/client_load_reporting_filter.h | 29 ---- .../client_channel/lb_policy/xds/xds.cc | 27 +--- ...lancer_api.cc => xds_load_balancer_api.cc} | 2 +- ...balancer_api.h => xds_load_balancer_api.h} | 6 +- .../plugin_registry/grpc_plugin_registry.cc | 4 + .../grpc_unsecure_plugin_registry.cc | 4 + src/python/grpcio/grpc_core_dependencies.py | 6 +- tools/doxygen/Doxyfile.core.internal | 8 + .../generated/sources_and_headers.json | 101 ++++++------- 21 files changed, 211 insertions(+), 342 deletions(-) delete mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc delete mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h rename src/core/ext/filters/client_channel/lb_policy/xds/{load_balancer_api.cc => xds_load_balancer_api.cc} (99%) rename src/core/ext/filters/client_channel/lb_policy/xds/{load_balancer_api.h => xds_load_balancer_api.h} (92%) diff --git a/BUILD b/BUILD index 71d7b7d09ef..4a6928ac4f5 100644 --- a/BUILD +++ b/BUILD @@ -279,6 +279,7 @@ grpc_cc_library( deps = [ "grpc_common", "grpc_lb_policy_grpclb", + "grpc_lb_policy_xds", ], ) @@ -294,6 +295,7 @@ grpc_cc_library( deps = [ "grpc_common", "grpc_lb_policy_grpclb_secure", + "grpc_lb_policy_xds_secure", "grpc_secure", "grpc_transport_chttp2_client_secure", "grpc_transport_chttp2_server_secure", @@ -1198,6 +1200,24 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpclb_proto", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + ], + hdrs = [ + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", + ], + external_deps = [ + "nanopb", + ], + language = "c++", +) + grpc_cc_library( name = "grpc_lb_policy_grpclb", srcs = [ @@ -1206,9 +1226,6 @@ grpc_cc_library( "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h", @@ -1216,9 +1233,6 @@ grpc_cc_library( "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", ], external_deps = [ "nanopb", @@ -1228,6 +1242,7 @@ grpc_cc_library( "grpc_base", "grpc_client_channel", "grpc_resolver_fake", + "grpclb_proto", ], ) @@ -1239,9 +1254,6 @@ grpc_cc_library( "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h", @@ -1249,9 +1261,6 @@ grpc_cc_library( "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", ], external_deps = [ "nanopb", @@ -1262,30 +1271,23 @@ grpc_cc_library( "grpc_client_channel", "grpc_resolver_fake", "grpc_secure", + "grpclb_proto", ], ) grpc_cc_library( name = "grpc_lb_policy_xds", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc", ], hdrs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h", ], external_deps = [ "nanopb", @@ -1295,30 +1297,23 @@ grpc_cc_library( "grpc_base", "grpc_client_channel", "grpc_resolver_fake", + "grpclb_proto", ], ) grpc_cc_library( name = "grpc_lb_policy_xds_secure", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc", ], hdrs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h", ], external_deps = [ "nanopb", @@ -1329,6 +1324,7 @@ grpc_cc_library( "grpc_client_channel", "grpc_resolver_fake", "grpc_secure", + "grpclb_proto", ], ) @@ -1586,6 +1582,7 @@ grpc_cc_library( ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/lib/security/context/security_context.h", "src/core/lib/security/credentials/alts/alts_credentials.h", "src/core/lib/security/credentials/composite/composite_credentials.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index e8de383ab10..e56c64abde6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1264,10 +1264,14 @@ add_library(grpc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c - src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -2617,12 +2621,16 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c third_party/nanopb/pb_common.c third_party/nanopb/pb_decode.c third_party/nanopb/pb_encode.c + src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c + src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c + src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c + src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc + src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc src/core/ext/filters/census/grpc_context.cc diff --git a/Makefile b/Makefile index b2f28f8a6dd..fe320fa9dc0 100644 --- a/Makefile +++ b/Makefile @@ -3732,10 +3732,14 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ + src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c \ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c \ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ - src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ @@ -5031,12 +5035,16 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c \ - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c \ - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ + src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c \ + src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c \ + src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ + src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/census/grpc_context.cc \ @@ -24804,6 +24812,7 @@ ifneq ($(OPENSSL_DEP),) # installing headers to their final destination on the drive. We need this # otherwise parallel compilation will fail if a source is compiled first. src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc: $(OPENSSL_DEP) +src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc: $(OPENSSL_DEP) src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc: $(OPENSSL_DEP) src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc: $(OPENSSL_DEP) src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index b2011562a24..9386048e21d 100644 --- a/build.yaml +++ b/build.yaml @@ -655,24 +655,19 @@ filegroups: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h src: - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c plugin: grpc_lb_policy_grpclb uses: - grpc_base - grpc_client_channel - nanopb - grpc_resolver_fake + - grpclb_proto - name: grpc_lb_policy_grpclb_secure headers: - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h @@ -680,18 +675,12 @@ filegroups: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h src: - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c plugin: grpc_lb_policy_grpclb uses: - grpc_base @@ -699,6 +688,7 @@ filegroups: - grpc_client_channel - nanopb - grpc_resolver_fake + - grpclb_proto - name: grpc_lb_policy_pick_first src: - src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -717,48 +707,33 @@ filegroups: - grpc_lb_subchannel_list - name: grpc_lb_policy_xds headers: - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h - - src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h - - src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h - src/core/ext/filters/client_channel/lb_policy/xds/xds.h - src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h - src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h + - src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h src: - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c - - src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc - - src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc + - src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc plugin: grpc_lb_policy_xds uses: - grpc_base - grpc_client_channel - nanopb - grpc_resolver_fake + - grpclb_proto - name: grpc_lb_policy_xds_secure headers: - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h - - src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h - - src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h - src/core/ext/filters/client_channel/lb_policy/xds/xds.h - src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h - src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h + - src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h src: - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c - - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c - - src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc - - src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc + - src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc plugin: grpc_lb_policy_xds uses: - grpc_base @@ -766,6 +741,7 @@ filegroups: - grpc_client_channel - nanopb - grpc_resolver_fake + - grpclb_proto - name: grpc_lb_subchannel_list headers: - src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -833,6 +809,7 @@ filegroups: - include/grpc/grpc_security.h headers: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h + - src/core/ext/filters/client_channel/lb_policy/xds/xds.h - src/core/lib/security/context/security_context.h - src/core/lib/security/credentials/alts/alts_credentials.h - src/core/lib/security/credentials/composite/composite_credentials.h @@ -1119,6 +1096,17 @@ filegroups: uses: - grpc_base - grpc_server_backward_compatibility +- name: grpclb_proto + headers: + - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h + - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h + - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h + src: + - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c + - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c + - src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c + uses: + - nanopb - name: nanopb src: - third_party/nanopb/pb_common.c @@ -1514,6 +1502,7 @@ libs: - grpc_transport_chttp2_client_insecure - grpc_transport_inproc - grpc_lb_policy_grpclb_secure + - grpc_lb_policy_xds_secure - grpc_lb_policy_pick_first - grpc_lb_policy_round_robin - grpc_resolver_dns_ares @@ -1593,6 +1582,7 @@ libs: - grpc_resolver_sockaddr - grpc_resolver_fake - grpc_lb_policy_grpclb + - grpc_lb_policy_xds - grpc_lb_policy_pick_first - grpc_lb_policy_round_robin - census diff --git a/config.m4 b/config.m4 index b9862f34ab7..9b758b3f140 100644 --- a/config.m4 +++ b/config.m4 @@ -374,10 +374,14 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ + src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c \ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c \ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ - src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ @@ -668,6 +672,7 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/pick_first) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/round_robin) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/xds) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns/c_ares) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns/native) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/fake) diff --git a/config.w32 b/config.w32 index 90a51057c42..a1bada1c616 100644 --- a/config.w32 +++ b/config.w32 @@ -349,10 +349,14 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\grpclb_channel_secure.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\grpclb_client_stats.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\load_balancer_api.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\fake\\fake_resolver.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\proto\\grpc\\lb\\v1\\google\\protobuf\\duration.pb.c " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\proto\\grpc\\lb\\v1\\google\\protobuf\\timestamp.pb.c " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\proto\\grpc\\lb\\v1\\load_balancer.pb.c " + - "src\\core\\ext\\filters\\client_channel\\resolver\\fake\\fake_resolver.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_channel_secure.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_client_stats.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_load_balancer_api.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first\\pick_first.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin\\round_robin.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\dns_resolver_ares.cc " + @@ -678,6 +682,7 @@ if (PHP_GRPC != "no") { FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\proto\\grpc\\lb\\v1\\google\\protobuf"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\xds"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver\\dns"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares"); diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 3cc19d5908b..1ab17006e38 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -269,6 +269,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/http/message_compress/message_compress_filter.h', 'src/core/ext/filters/http/server/http_server_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/composite/composite_credentials.h', @@ -495,10 +496,13 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h', + 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h', 'src/core/ext/filters/client_channel/lb_policy/subchannel_list.h', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 8bb2311414b..8d63f67c347 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -276,6 +276,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/http/message_compress/message_compress_filter.h', 'src/core/ext/filters/http/server/http_server_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/composite/composite_credentials.h', @@ -502,10 +503,13 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h', + 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h', 'src/core/ext/filters/client_channel/lb_policy/subchannel_list.h', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h', @@ -802,10 +806,14 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', + 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', @@ -877,6 +885,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/http/message_compress/message_compress_filter.h', 'src/core/ext/filters/http/server/http_server_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/composite/composite_credentials.h', @@ -1103,10 +1112,13 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h', + 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h', 'src/core/ext/filters/client_channel/lb_policy/subchannel_list.h', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h', diff --git a/grpc.gemspec b/grpc.gemspec index 81f93f44197..533bb90b75c 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -208,6 +208,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/http/message_compress/message_compress_filter.h ) s.files += %w( src/core/ext/filters/http/server/http_server_filter.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds.h ) s.files += %w( src/core/lib/security/context/security_context.h ) s.files += %w( src/core/lib/security/credentials/alts/alts_credentials.h ) s.files += %w( src/core/lib/security/credentials/composite/composite_credentials.h ) @@ -438,10 +439,13 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h ) + s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h ) - s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/subchannel_list.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h ) @@ -741,10 +745,14 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c ) - s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc ) diff --git a/grpc.gyp b/grpc.gyp index 8e14b49cc4a..36a458b1391 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -566,10 +566,14 @@ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', + 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', @@ -1299,12 +1303,16 @@ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', 'third_party/nanopb/pb_decode.c', 'third_party/nanopb/pb_encode.c', + 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c', + 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c', + 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', 'src/core/ext/filters/census/grpc_context.cc', diff --git a/package.xml b/package.xml index be3fd0d4c2b..0d407077867 100644 --- a/package.xml +++ b/package.xml @@ -213,6 +213,7 @@ + @@ -443,10 +444,13 @@ + - + + + @@ -746,10 +750,14 @@ + - + + + + diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc b/src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc deleted file mode 100644 index d79453d0080..00000000000 --- a/src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc +++ /dev/null @@ -1,140 +0,0 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h" - -#include -#include - -#include "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/profiling/timers.h" - -static grpc_error* init_channel_elem(grpc_channel_element* elem, - grpc_channel_element_args* args) { - return GRPC_ERROR_NONE; -} - -static void destroy_channel_elem(grpc_channel_element* elem) {} - -namespace { - -struct call_data { - // Stats object to update. - grpc_core::RefCountedPtr client_stats; - // State for intercepting send_initial_metadata. - grpc_closure on_complete_for_send; - grpc_closure* original_on_complete_for_send; - bool send_initial_metadata_succeeded; - // State for intercepting recv_initial_metadata. - grpc_closure recv_initial_metadata_ready; - grpc_closure* original_recv_initial_metadata_ready; - bool recv_initial_metadata_succeeded; -}; - -} // namespace - -static void on_complete_for_send(void* arg, grpc_error* error) { - call_data* calld = static_cast(arg); - if (error == GRPC_ERROR_NONE) { - calld->send_initial_metadata_succeeded = true; - } - GRPC_CLOSURE_RUN(calld->original_on_complete_for_send, GRPC_ERROR_REF(error)); -} - -static void recv_initial_metadata_ready(void* arg, grpc_error* error) { - call_data* calld = static_cast(arg); - if (error == GRPC_ERROR_NONE) { - calld->recv_initial_metadata_succeeded = true; - } - GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, - GRPC_ERROR_REF(error)); -} - -static grpc_error* init_call_elem(grpc_call_element* elem, - const grpc_call_element_args* args) { - call_data* calld = static_cast(elem->call_data); - // Get stats object from context and take a ref. - GPR_ASSERT(args->context != nullptr); - if (args->context[GRPC_GRPCLB_CLIENT_STATS].value != nullptr) { - calld->client_stats = static_cast( - args->context[GRPC_GRPCLB_CLIENT_STATS].value) - ->Ref(); - // Record call started. - calld->client_stats->AddCallStarted(); - } - return GRPC_ERROR_NONE; -} - -static void destroy_call_elem(grpc_call_element* elem, - const grpc_call_final_info* final_info, - grpc_closure* ignored) { - call_data* calld = static_cast(elem->call_data); - if (calld->client_stats != nullptr) { - // Record call finished, optionally setting client_failed_to_send and - // received. - calld->client_stats->AddCallFinished( - !calld->send_initial_metadata_succeeded /* client_failed_to_send */, - calld->recv_initial_metadata_succeeded /* known_received */); - // All done, so unref the stats object. - // TODO(roth): Eliminate this once filter stack is converted to C++. - calld->client_stats.reset(); - } -} - -static void start_transport_stream_op_batch( - grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { - call_data* calld = static_cast(elem->call_data); - GPR_TIMER_SCOPE("clr_start_transport_stream_op_batch", 0); - if (calld->client_stats != nullptr) { - // Intercept send_initial_metadata. - if (batch->send_initial_metadata) { - calld->original_on_complete_for_send = batch->on_complete; - GRPC_CLOSURE_INIT(&calld->on_complete_for_send, on_complete_for_send, - calld, grpc_schedule_on_exec_ctx); - batch->on_complete = &calld->on_complete_for_send; - } - // Intercept recv_initial_metadata. - if (batch->recv_initial_metadata) { - calld->original_recv_initial_metadata_ready = - batch->payload->recv_initial_metadata.recv_initial_metadata_ready; - GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready, - recv_initial_metadata_ready, calld, - grpc_schedule_on_exec_ctx); - batch->payload->recv_initial_metadata.recv_initial_metadata_ready = - &calld->recv_initial_metadata_ready; - } - } - // Chain to next filter. - grpc_call_next_op(elem, batch); -} - -const grpc_channel_filter xds_client_load_reporting_filter = { - start_transport_stream_op_batch, - grpc_channel_next_op, - sizeof(call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - 0, // sizeof(channel_data) - init_channel_elem, - destroy_channel_elem, - grpc_channel_next_get_info, - "client_load_reporting"}; diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h b/src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h deleted file mode 100644 index 31a799154c0..00000000000 --- a/src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_CLIENT_LOAD_REPORTING_FILTER_H -#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_CLIENT_LOAD_REPORTING_FILTER_H - -#include - -#include "src/core/lib/channel/channel_stack.h" - -extern const grpc_channel_filter xds_client_load_reporting_filter; - -#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_CLIENT_LOAD_REPORTING_FILTER_H \ - */ diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index 16e910b2d9b..7fb4cbdcd28 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -75,11 +75,10 @@ #include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/ext/filters/client_channel/client_channel_factory.h" -#include "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h" -#include "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h" #include "src/core/ext/filters/client_channel/lb_policy/xds/xds.h" #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h" #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h" +#include "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" #include "src/core/ext/filters/client_channel/lb_policy_factory.h" #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/parse_address.h" @@ -1483,7 +1482,6 @@ void XdsLb::OnBalancerChannelConnectivityChangedLocked(void* arg, xdslb_policy->lb_call_backoff_.Reset(); xdslb_policy->StartBalancerCallLocked(); } - [[fallthrough]]; // Fall through. case GRPC_CHANNEL_SHUTDOWN: done: @@ -1861,34 +1859,11 @@ class XdsFactory : public LoadBalancingPolicyFactory { // Plugin registration // -namespace { - -// Only add client_load_reporting filter if the grpclb LB policy is used. -bool maybe_add_client_load_reporting_filter(grpc_channel_stack_builder* builder, - void* arg) { - const grpc_channel_args* args = - grpc_channel_stack_builder_get_channel_arguments(builder); - const grpc_arg* channel_arg = - grpc_channel_args_find(args, GRPC_ARG_LB_POLICY_NAME); - if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_STRING && - strcmp(channel_arg->value.string, "grpclb") == 0) { - return grpc_channel_stack_builder_append_filter( - builder, (const grpc_channel_filter*)arg, nullptr, nullptr); - } - return true; -} - -} // namespace - void grpc_lb_policy_xds_init() { grpc_core::LoadBalancingPolicyRegistry::Builder:: RegisterLoadBalancingPolicyFactory( grpc_core::UniquePtr( grpc_core::New())); - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_client_load_reporting_filter, - (void*)&xds_client_load_reporting_filter); } void grpc_lb_policy_xds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc similarity index 99% rename from src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc rename to src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc index 3bd26536355..79b7bdbe338 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc @@ -20,7 +20,7 @@ #include "pb_decode.h" #include "pb_encode.h" -#include "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h" +#include "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" #include diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h b/src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h similarity index 92% rename from src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h rename to src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h index 3f096af6d6f..9d08defa7ef 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h @@ -16,8 +16,8 @@ * */ -#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_LOAD_BALANCER_API_H -#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_LOAD_BALANCER_API_H +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_LOAD_BALANCER_API_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_LOAD_BALANCER_API_H #include @@ -85,5 +85,5 @@ grpc_millis xds_grpclb_duration_to_millis(xds_grpclb_duration* duration_pb); /** Destroy \a initial_response */ void xds_grpclb_initial_response_destroy(xds_grpclb_initial_response* response); -#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_LOAD_BALANCER_API_H \ +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_LOAD_BALANCER_API_H \ */ diff --git a/src/core/plugin_registry/grpc_plugin_registry.cc b/src/core/plugin_registry/grpc_plugin_registry.cc index fb523a173de..94c2493d5e5 100644 --- a/src/core/plugin_registry/grpc_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_plugin_registry.cc @@ -36,6 +36,8 @@ void grpc_resolver_fake_init(void); void grpc_resolver_fake_shutdown(void); void grpc_lb_policy_grpclb_init(void); void grpc_lb_policy_grpclb_shutdown(void); +void grpc_lb_policy_xds_init(void); +void grpc_lb_policy_xds_shutdown(void); void grpc_lb_policy_pick_first_init(void); void grpc_lb_policy_pick_first_shutdown(void); void grpc_lb_policy_round_robin_init(void); @@ -72,6 +74,8 @@ void grpc_register_built_in_plugins(void) { grpc_resolver_fake_shutdown); grpc_register_plugin(grpc_lb_policy_grpclb_init, grpc_lb_policy_grpclb_shutdown); + grpc_register_plugin(grpc_lb_policy_xds_init, + grpc_lb_policy_xds_shutdown); grpc_register_plugin(grpc_lb_policy_pick_first_init, grpc_lb_policy_pick_first_shutdown); grpc_register_plugin(grpc_lb_policy_round_robin_init, diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc index 80214aebe2e..5749ab6b954 100644 --- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc @@ -40,6 +40,8 @@ void grpc_resolver_fake_init(void); void grpc_resolver_fake_shutdown(void); void grpc_lb_policy_grpclb_init(void); void grpc_lb_policy_grpclb_shutdown(void); +void grpc_lb_policy_xds_init(void); +void grpc_lb_policy_xds_shutdown(void); void grpc_lb_policy_pick_first_init(void); void grpc_lb_policy_pick_first_shutdown(void); void grpc_lb_policy_round_robin_init(void); @@ -74,6 +76,8 @@ void grpc_register_built_in_plugins(void) { grpc_resolver_fake_shutdown); grpc_register_plugin(grpc_lb_policy_grpclb_init, grpc_lb_policy_grpclb_shutdown); + grpc_register_plugin(grpc_lb_policy_xds_init, + grpc_lb_policy_xds_shutdown); grpc_register_plugin(grpc_lb_policy_pick_first_init, grpc_lb_policy_pick_first_shutdown); grpc_register_plugin(grpc_lb_policy_round_robin_init, diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 932621a38dd..0bd3951a19d 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -348,10 +348,14 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', + 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index d5986fde90b..204ef493bac 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -910,6 +910,14 @@ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balan src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/lb_policy/subchannel_list.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy_factory.cc \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6e1eb6e3c0f..20b6d36671e 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7047,6 +7047,7 @@ "grpc_lb_policy_grpclb_secure", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", + "grpc_lb_policy_xds_secure", "grpc_max_age_filter", "grpc_message_size_filter", "grpc_resolver_dns_ares", @@ -7140,6 +7141,7 @@ "grpc_lb_policy_grpclb", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", + "grpc_lb_policy_xds", "grpc_max_age_filter", "grpc_message_size_filter", "grpc_resolver_dns_ares", @@ -10137,6 +10139,7 @@ "grpc_base", "grpc_client_channel", "grpc_resolver_fake", + "grpclb_proto", "nanopb" ], "headers": [ @@ -10144,10 +10147,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "is_filegroup": true, "language": "c", @@ -10162,13 +10162,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "third_party": false, "type": "filegroup" @@ -10180,6 +10174,7 @@ "grpc_client_channel", "grpc_resolver_fake", "grpc_secure", + "grpclb_proto", "nanopb" ], "headers": [ @@ -10187,10 +10182,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "is_filegroup": true, "language": "c", @@ -10205,13 +10197,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "third_party": false, "type": "filegroup" @@ -10256,38 +10242,27 @@ "grpc_base", "grpc_client_channel", "grpc_resolver_fake", + "grpclb_proto", "nanopb" ], "headers": [ - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", - "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h" + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" ], "is_filegroup": true, "language": "c", "name": "grpc_lb_policy_xds", "src": [ - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h" + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" ], "third_party": false, "type": "filegroup" @@ -10299,38 +10274,27 @@ "grpc_client_channel", "grpc_resolver_fake", "grpc_secure", + "grpclb_proto", "nanopb" ], "headers": [ - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", - "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h" + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" ], "is_filegroup": true, "language": "c", "name": "grpc_lb_policy_xds_secure", "src": [ - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/client_load_reporting_filter.h", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/load_balancer_api.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h" + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" ], "third_party": false, "type": "filegroup" @@ -10480,6 +10444,7 @@ "headers": [ "include/grpc/grpc_security.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/lib/security/context/security_context.h", "src/core/lib/security/credentials/alts/alts_credentials.h", "src/core/lib/security/credentials/composite/composite_credentials.h", @@ -10512,6 +10477,7 @@ "src": [ "include/grpc/grpc_security.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/lib/http/httpcli_security_connector.cc", "src/core/lib/security/context/security_context.cc", "src/core/lib/security/context/security_context.h", @@ -11006,6 +10972,29 @@ "third_party": false, "type": "filegroup" }, + { + "deps": [ + "nanopb" + ], + "headers": [ + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpclb_proto", + "src": [ + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + ], + "third_party": false, + "type": "filegroup" + }, { "deps": [ "nanopb_headers" From 3a9411ca1e733ce9f87b22136014c7a00bdeacbd Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Thu, 18 Oct 2018 15:49:42 -0400 Subject: [PATCH 42/53] Optimize error handling for special cases. This commit contains a few improvements: 1. Using a consequetive range of [0..4], will allow us to merge all branches of error_is_special into one comparison. 2. With (1), we can remove the for loops to find entries in error_status_map with a single O(1) lookup. 3. grpc_error_is_special() code paths should be inlined for ref and unref to avoid callq for the majority of cases where speical error is used. 4. grpc_error_get_int() should never accept a nullptr argument to avoid an expensive branch in the hot path. Callers should all allocate a dummy int on the stack when calling. --- .../ext/transport/chttp2/transport/parsing.cc | 6 ++- src/core/lib/iomgr/error.cc | 47 +++++++------------ src/core/lib/iomgr/error.h | 31 ++++++++++-- src/core/lib/iomgr/error_internal.h | 2 - src/core/lib/transport/error_utils.cc | 6 ++- 5 files changed, 51 insertions(+), 41 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index f532b084c93..1ff96d3cd36 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -368,6 +368,7 @@ static grpc_error* init_data_frame_parser(grpc_chttp2_transport* t) { &s->data_parser, t->incoming_frame_flags, s->id, s); } error_handler: + intptr_t unused; if (err == GRPC_ERROR_NONE) { t->incoming_stream = s; /* t->parser = grpc_chttp2_data_parser_parse;*/ @@ -375,7 +376,7 @@ error_handler: t->parser_data = &s->data_parser; t->ping_state.last_ping_sent_time = GRPC_MILLIS_INF_PAST; return GRPC_ERROR_NONE; - } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, nullptr)) { + } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, &unused)) { /* handle stream errors by closing the stream */ if (s != nullptr) { grpc_chttp2_mark_stream_closed(t, s, true, false, err); @@ -756,9 +757,10 @@ static grpc_error* parse_frame_slice(grpc_chttp2_transport* t, grpc_slice slice, int is_last) { grpc_chttp2_stream* s = t->incoming_stream; grpc_error* err = t->parser(t->parser_data, t, s, slice, is_last); + intptr_t unused; if (GPR_LIKELY(err == GRPC_ERROR_NONE)) { return err; - } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, nullptr)) { + } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, &unused)) { if (grpc_http_trace.enabled()) { const char* msg = grpc_error_string(err); gpr_log(GPR_ERROR, "%s", msg); diff --git a/src/core/lib/iomgr/error.cc b/src/core/lib/iomgr/error.cc index 146a539027d..6ae077fd548 100644 --- a/src/core/lib/iomgr/error.cc +++ b/src/core/lib/iomgr/error.cc @@ -121,14 +121,8 @@ static const char* error_time_name(grpc_error_times key) { GPR_UNREACHABLE_CODE(return "unknown"); } -bool grpc_error_is_special(grpc_error* err) { - return err == GRPC_ERROR_NONE || err == GRPC_ERROR_OOM || - err == GRPC_ERROR_CANCELLED; -} - #ifndef NDEBUG -grpc_error* grpc_error_ref(grpc_error* err, const char* file, int line) { - if (grpc_error_is_special(err)) return err; +grpc_error* grpc_error_do_ref(grpc_error* err, const char* file, int line) { if (grpc_trace_error_refcount.enabled()) { gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d]", err, gpr_atm_no_barrier_load(&err->atomics.refs.count), @@ -138,8 +132,7 @@ grpc_error* grpc_error_ref(grpc_error* err, const char* file, int line) { return err; } #else -grpc_error* grpc_error_ref(grpc_error* err) { - if (grpc_error_is_special(err)) return err; +grpc_error* grpc_error_do_ref(grpc_error* err) { gpr_ref(&err->atomics.refs); return err; } @@ -177,8 +170,7 @@ static void error_destroy(grpc_error* err) { } #ifndef NDEBUG -void grpc_error_unref(grpc_error* err, const char* file, int line) { - if (grpc_error_is_special(err)) return; +void grpc_error_do_unref(grpc_error* err, const char* file, int line) { if (grpc_trace_error_refcount.enabled()) { gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d]", err, gpr_atm_no_barrier_load(&err->atomics.refs.count), @@ -189,8 +181,7 @@ void grpc_error_unref(grpc_error* err, const char* file, int line) { } } #else -void grpc_error_unref(grpc_error* err) { - if (grpc_error_is_special(err)) return; +void grpc_error_do_unref(grpc_error* err) { if (gpr_unref(&err->atomics.refs)) { error_destroy(err); } @@ -450,26 +441,23 @@ grpc_error* grpc_error_set_int(grpc_error* src, grpc_error_ints which, } typedef struct { - grpc_error* error; grpc_status_code code; const char* msg; } special_error_status_map; static const special_error_status_map error_status_map[] = { - {GRPC_ERROR_NONE, GRPC_STATUS_OK, ""}, - {GRPC_ERROR_CANCELLED, GRPC_STATUS_CANCELLED, "Cancelled"}, - {GRPC_ERROR_OOM, GRPC_STATUS_RESOURCE_EXHAUSTED, "Out of memory"}, + {GRPC_STATUS_OK, ""}, // GRPC_ERROR_NONE + {GRPC_STATUS_INVALID_ARGUMENT, ""}, // GRPC_ERROR_RESERVED_1 + {GRPC_STATUS_RESOURCE_EXHAUSTED, "Out of memory"}, // GRPC_ERROR_OOM + {GRPC_STATUS_INVALID_ARGUMENT, ""}, // GRPC_ERROR_RESERVED_2 + {GRPC_STATUS_CANCELLED, "Cancelled"}, // GRPC_ERROR_CANCELLED }; bool grpc_error_get_int(grpc_error* err, grpc_error_ints which, intptr_t* p) { GPR_TIMER_SCOPE("grpc_error_get_int", 0); if (grpc_error_is_special(err)) { - for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { - if (error_status_map[i].error == err) { - if (which != GRPC_ERROR_INT_GRPC_STATUS) return false; - if (p != nullptr) *p = error_status_map[i].code; - return true; - } - } + if (which != GRPC_ERROR_INT_GRPC_STATUS) return false; + *p = error_status_map[reinterpret_cast(err)].code; + return true; } uint8_t slot = err->ints[which]; if (slot != UINT8_MAX) { @@ -490,13 +478,10 @@ grpc_error* grpc_error_set_str(grpc_error* src, grpc_error_strs which, bool grpc_error_get_str(grpc_error* err, grpc_error_strs which, grpc_slice* str) { if (grpc_error_is_special(err)) { - for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { - if (error_status_map[i].error == err) { - if (which != GRPC_ERROR_STR_GRPC_MESSAGE) return false; - *str = grpc_slice_from_static_string(error_status_map[i].msg); - return true; - } - } + if (which != GRPC_ERROR_STR_GRPC_MESSAGE) return false; + *str = grpc_slice_from_static_string( + error_status_map[reinterpret_cast(err)].msg); + return true; } uint8_t slot = err->strs[which]; if (slot != UINT8_MAX) { diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h index 49f4029bc22..6978ef6d91e 100644 --- a/src/core/lib/iomgr/error.h +++ b/src/core/lib/iomgr/error.h @@ -120,8 +120,15 @@ typedef enum { /// polling engines) can safely use the lower bit for themselves. #define GRPC_ERROR_NONE ((grpc_error*)NULL) +#define GRPC_ERROR_RESERVED_1 ((grpc_error*)1) #define GRPC_ERROR_OOM ((grpc_error*)2) +#define GRPC_ERROR_RESERVED_2 ((grpc_error*)3) #define GRPC_ERROR_CANCELLED ((grpc_error*)4) +#define GRPC_ERROR_SPECIAL_MAX GRPC_ERROR_CANCELLED + +inline bool grpc_error_is_special(struct grpc_error* err) { + return err <= GRPC_ERROR_SPECIAL_MAX; +} // debug only toggles that allow for a sanity to check that ensures we will // never create any errors in the per-RPC hotpath. @@ -158,13 +165,29 @@ grpc_error* grpc_error_create(const char* file, int line, grpc_slice desc, errs, count) #ifndef NDEBUG -grpc_error* grpc_error_ref(grpc_error* err, const char* file, int line); -void grpc_error_unref(grpc_error* err, const char* file, int line); +grpc_error* grpc_error_do_ref(grpc_error* err, const char* file, int line); +void grpc_error_do_unref(grpc_error* err, const char* file, int line); +inline grpc_error* grpc_error_ref(grpc_error* err, const char* file, int line) { + if (grpc_error_is_special(err)) return err; + return grpc_error_do_ref(err, file, line); +} +inline void grpc_error_unref(grpc_error* err, const char* file, int line) { + if (grpc_error_is_special(err)) return; + grpc_error_do_unref(err, file, line); +} #define GRPC_ERROR_REF(err) grpc_error_ref(err, __FILE__, __LINE__) #define GRPC_ERROR_UNREF(err) grpc_error_unref(err, __FILE__, __LINE__) #else -grpc_error* grpc_error_ref(grpc_error* err); -void grpc_error_unref(grpc_error* err); +grpc_error* grpc_error_do_ref(grpc_error* err); +void grpc_error_do_unref(grpc_error* err); +inline grpc_error* grpc_error_ref(grpc_error* err) { + if (grpc_error_is_special(err)) return err; + return grpc_error_do_ref(err); +} +inline void grpc_error_unref(grpc_error* err) { + if (grpc_error_is_special(err)) return; + grpc_error_do_unref(err); +} #define GRPC_ERROR_REF(err) grpc_error_ref(err) #define GRPC_ERROR_UNREF(err) grpc_error_unref(err) #endif diff --git a/src/core/lib/iomgr/error_internal.h b/src/core/lib/iomgr/error_internal.h index 7fde347abd0..80273960198 100644 --- a/src/core/lib/iomgr/error_internal.h +++ b/src/core/lib/iomgr/error_internal.h @@ -58,6 +58,4 @@ struct grpc_error { intptr_t arena[0]; }; -bool grpc_error_is_special(struct grpc_error* err); - #endif /* GRPC_CORE_LIB_IOMGR_ERROR_INTERNAL_H */ diff --git a/src/core/lib/transport/error_utils.cc b/src/core/lib/transport/error_utils.cc index 2eff8b29167..558f1d494cd 100644 --- a/src/core/lib/transport/error_utils.cc +++ b/src/core/lib/transport/error_utils.cc @@ -26,8 +26,9 @@ static grpc_error* recursively_find_error_with_field(grpc_error* error, grpc_error_ints which) { + intptr_t unused; // If the error itself has a status code, return it. - if (grpc_error_get_int(error, which, nullptr)) { + if (grpc_error_get_int(error, which, &unused)) { return error; } if (grpc_error_is_special(error)) return nullptr; @@ -102,7 +103,8 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline, } bool grpc_error_has_clear_grpc_status(grpc_error* error) { - if (grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, nullptr)) { + intptr_t unused; + if (grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &unused)) { return true; } uint8_t slot = error->first_err; From 24b6d7194f85892f09c7104924ce607ff231af6f Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Thu, 18 Oct 2018 16:37:07 -0400 Subject: [PATCH 43/53] Add comment for grpc_error_get_int(). Explain that the input parameter cannot be nullptr. --- src/core/lib/iomgr/error.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h index 6978ef6d91e..cb740d5b01c 100644 --- a/src/core/lib/iomgr/error.h +++ b/src/core/lib/iomgr/error.h @@ -194,6 +194,8 @@ inline void grpc_error_unref(grpc_error* err) { grpc_error* grpc_error_set_int(grpc_error* src, grpc_error_ints which, intptr_t value) GRPC_MUST_USE_RESULT; +/// It is an error to pass nullptr as `p`. Caller should allocate a dummy +/// intptr_t for `p`, even if the value of `p` is not used. bool grpc_error_get_int(grpc_error* error, grpc_error_ints which, intptr_t* p); /// This call takes ownership of the slice; the error is responsible for /// eventually unref-ing it. From 1a7f14fd310fafc3df9a06d161f7cebc2e9ad7ad Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 18 Oct 2018 14:00:09 -0700 Subject: [PATCH 44/53] Remove uneeded header --- test/cpp/microbenchmarks/BUILD | 1 - test/cpp/microbenchmarks/bm_call_create.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 6c1859ff4fe..097e92f5836 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -41,7 +41,6 @@ grpc_cc_library( ], deps = [ "//:grpc++_unsecure", - "//:lb_server_load_reporting_filter", "//src/proto/grpc/testing:echo_proto", "//test/core/util:grpc_test_util_unsecure", "//test/cpp/util:test_config", diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 389b888084f..1f7831096c1 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -34,7 +34,6 @@ #include "src/core/ext/filters/http/client/http_client_filter.h" #include "src/core/ext/filters/http/message_compress/message_compress_filter.h" #include "src/core/ext/filters/http/server/http_server_filter.h" -#include "src/core/ext/filters/load_reporting/server_load_reporting_filter.h" #include "src/core/ext/filters/message_size/message_size_filter.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/connected_channel.h" From 339e7f4a82fa57a134f2cca3ce0773fcc2b1f3d8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 18 Oct 2018 14:08:03 +0200 Subject: [PATCH 45/53] upload to BQ from singlemachine benchmarks too --- tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh b/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh index 86d669be33f..3c9798d375e 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh @@ -25,4 +25,5 @@ tools/run_tests/run_performance_tests.py \ --netperf \ --category smoketest \ -u kbuilder \ + --bq_result_table performance_test.performance_experiment_singlevm \ --xml_report reports/singlemachine/sponge_log.xml From 8c08e6af53664d2ceb5aff40546092492ee21112 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 18 Oct 2018 14:54:29 +0200 Subject: [PATCH 46/53] script for creating peformance workers from image --- ...ux_kokoro_performance_worker_from_image.sh | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 tools/gce/create_linux_kokoro_performance_worker_from_image.sh diff --git a/tools/gce/create_linux_kokoro_performance_worker_from_image.sh b/tools/gce/create_linux_kokoro_performance_worker_from_image.sh new file mode 100755 index 00000000000..0f7939be4c0 --- /dev/null +++ b/tools/gce/create_linux_kokoro_performance_worker_from_image.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Creates a performance worker on GCE from an image that's used for kokoro +# perf workers. + +set -ex + +cd "$(dirname "$0")" + +CLOUD_PROJECT=grpc-testing +ZONE=us-central1-b # this zone allows 32core machines +LATEST_PERF_WORKER_IMAGE=grpc-performance-kokoro-v2 # update if newer image exists + +INSTANCE_NAME="${1:-grpc-kokoro-performance-server}" +MACHINE_TYPE="${2:-n1-standard-32}" + +gcloud compute instances create "$INSTANCE_NAME" \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + --machine-type "$MACHINE_TYPE" \ + --image-project "$CLOUD_PROJECT" \ + --image "$LATEST_PERF_WORKER_IMAGE" \ + --boot-disk-size 300 \ + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh From 3977c30f475469206833cc6fc1b811f3cc404312 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 19 Oct 2018 17:38:23 +0200 Subject: [PATCH 47/53] add a note message length encoding --- doc/PROTOCOL-HTTP2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/PROTOCOL-HTTP2.md b/doc/PROTOCOL-HTTP2.md index bdd00ca3632..a354dad8636 100644 --- a/doc/PROTOCOL-HTTP2.md +++ b/doc/PROTOCOL-HTTP2.md @@ -92,7 +92,7 @@ The repeated sequence of **Length-Prefixed-Message** items is delivered in DATA * **Length-Prefixed-Message** → Compressed-Flag Message-Length Message * **Compressed-Flag** → 0 / 1 # encoded as 1 byte unsigned integer -* **Message-Length** → {_length of Message_} # encoded as 4 byte unsigned integer +* **Message-Length** → {_length of Message_} # encoded as 4 byte unsigned integer (big endian) * **Message** → \*{binary octet} A **Compressed-Flag** value of 1 indicates that the binary octet sequence of **Message** is compressed using the mechanism declared by the **Message-Encoding** header. A value of 0 indicates that no encoding of **Message** bytes has occurred. Compression contexts are NOT maintained over message boundaries, implementations must create a new context for each message in the stream. If the **Message-Encoding** header is omitted then the **Compressed-Flag** must be 0. From 0f4968a6d9244f3bcef89bbb1af60fdfcaf5d5b1 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 17 Oct 2018 11:29:45 -0700 Subject: [PATCH 48/53] Fix a dangling pointer on ares_request object in case of cancellation --- .../resolver/dns/c_ares/dns_resolver_ares.cc | 3 +- .../resolver/dns/c_ares/grpc_ares_wrapper.cc | 80 ++++++++++--------- .../resolver/dns/c_ares/grpc_ares_wrapper.h | 5 +- .../dns/c_ares/grpc_ares_wrapper_fallback.cc | 5 +- .../dns_resolver_connectivity_test.cc | 5 ++ test/core/end2end/fuzzers/api_fuzzer.cc | 5 ++ test/core/end2end/goaway_server_test.cc | 10 +++ 7 files changed, 70 insertions(+), 43 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index dfa52867d8b..cfb90eb5706 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -201,7 +201,7 @@ void AresDnsResolver::ShutdownLocked() { grpc_timer_cancel(&next_resolution_timer_); } if (pending_request_ != nullptr) { - grpc_cancel_ares_request(pending_request_); + grpc_cancel_ares_request_locked(pending_request_); } if (next_completion_ != nullptr) { *target_result_ = nullptr; @@ -298,6 +298,7 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) { grpc_channel_args* result = nullptr; GPR_ASSERT(r->resolving_); r->resolving_ = false; + gpr_free(r->pending_request_); r->pending_request_ = nullptr; if (r->lb_addresses_ != nullptr) { static const char* args_to_remove[2]; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index 4c795c34c8e..582e2203fc7 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -144,12 +144,12 @@ static void grpc_ares_request_unref_locked(grpc_ares_request* r) { void grpc_ares_complete_request_locked(grpc_ares_request* r) { /* Invoke on_done callback and destroy the request */ + r->ev_driver = nullptr; grpc_lb_addresses* lb_addrs = *(r->lb_addrs_out); if (lb_addrs != nullptr) { grpc_cares_wrapper_address_sorting_sort(lb_addrs); } GRPC_CLOSURE_SCHED(r->on_done, r->error); - gpr_free(r); } static grpc_ares_hostbyname_request* create_hostbyname_request_locked( @@ -356,15 +356,12 @@ done: grpc_ares_request_unref_locked(r); } -static grpc_ares_request* -grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( - const char* dns_server, const char* name, const char* default_port, - grpc_pollset_set* interested_parties, grpc_closure* on_done, - grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, - grpc_combiner* combiner) { +void grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( + grpc_ares_request* r, const char* dns_server, const char* name, + const char* default_port, grpc_pollset_set* interested_parties, + bool check_grpclb, grpc_combiner* combiner) { grpc_error* error = GRPC_ERROR_NONE; grpc_ares_hostbyname_request* hr = nullptr; - grpc_ares_request* r = nullptr; ares_channel* channel = nullptr; /* TODO(zyc): Enable tracing after #9603 is checked in */ /* if (grpc_dns_trace) { @@ -390,14 +387,6 @@ grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( } port = gpr_strdup(default_port); } - r = static_cast(gpr_zalloc(sizeof(grpc_ares_request))); - r->ev_driver = nullptr; - r->on_done = on_done; - r->lb_addrs_out = addrs; - r->service_config_json_out = service_config_json; - r->success = false; - r->error = GRPC_ERROR_NONE; - r->pending_queries = 0; error = grpc_ares_ev_driver_create_locked(&r->ev_driver, interested_parties, combiner, r); if (error != GRPC_ERROR_NONE) goto error_cleanup; @@ -458,7 +447,7 @@ grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( on_srv_query_done_locked, r); gpr_free(service_name); } - if (service_config_json != nullptr) { + if (r->service_config_json_out != nullptr) { grpc_ares_request_ref_locked(r); char* config_name; gpr_asprintf(&config_name, "_grpc_config.%s", host); @@ -470,14 +459,12 @@ grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( grpc_ares_request_unref_locked(r); gpr_free(host); gpr_free(port); - return r; + return; error_cleanup: - GRPC_CLOSURE_SCHED(on_done, error); - gpr_free(r); + GRPC_CLOSURE_SCHED(r->on_done, error); gpr_free(host); gpr_free(port); - return nullptr; } static bool inner_resolve_as_ip_literal_locked(const char* name, @@ -536,21 +523,31 @@ static grpc_ares_request* grpc_dns_lookup_ares_locked_impl( grpc_pollset_set* interested_parties, grpc_closure* on_done, grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, grpc_combiner* combiner) { + grpc_ares_request* r = + static_cast(gpr_zalloc(sizeof(grpc_ares_request))); + r->ev_driver = nullptr; + r->on_done = on_done; + r->lb_addrs_out = addrs; + r->service_config_json_out = service_config_json; + r->success = false; + r->error = GRPC_ERROR_NONE; + r->pending_queries = 0; // Early out if the target is an ipv4 or ipv6 literal. if (resolve_as_ip_literal_locked(name, default_port, addrs)) { GRPC_CLOSURE_SCHED(on_done, GRPC_ERROR_NONE); - return nullptr; + return r; } // Early out if the target is localhost and we're on Windows. if (grpc_ares_maybe_resolve_localhost_manually_locked(name, default_port, addrs)) { GRPC_CLOSURE_SCHED(on_done, GRPC_ERROR_NONE); - return nullptr; + return r; } // Look up name using c-ares lib. - return grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( - dns_server, name, default_port, interested_parties, on_done, addrs, - check_grpclb, service_config_json, combiner); + grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( + r, dns_server, name, default_port, interested_parties, check_grpclb, + combiner); + return r; } grpc_ares_request* (*grpc_dns_lookup_ares_locked)( @@ -559,14 +556,16 @@ grpc_ares_request* (*grpc_dns_lookup_ares_locked)( grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, grpc_combiner* combiner) = grpc_dns_lookup_ares_locked_impl; -void grpc_cancel_ares_request(grpc_ares_request* r) { - if (grpc_dns_lookup_ares_locked == grpc_dns_lookup_ares_locked_impl) { - if (r != nullptr) { - grpc_ares_ev_driver_shutdown_locked(r->ev_driver); - } +static void grpc_cancel_ares_request_locked_impl(grpc_ares_request* r) { + GPR_ASSERT(r != nullptr); + if (r->ev_driver != nullptr) { + grpc_ares_ev_driver_shutdown_locked(r->ev_driver); } } +void (*grpc_cancel_ares_request_locked)(grpc_ares_request* r) = + grpc_cancel_ares_request_locked_impl; + grpc_error* grpc_ares_init(void) { gpr_once_init(&g_basic_init, do_basic_init); gpr_mu_lock(&g_init_mu); @@ -603,20 +602,23 @@ typedef struct grpc_resolve_address_ares_request { grpc_lb_addresses* lb_addrs; /** closure to call when the resolve_address_ares request completes */ grpc_closure* on_resolve_address_done; - /** a closure wrapping on_dns_lookup_done_cb, which should be invoked when the - grpc_dns_lookup_ares_locked operation is done. */ - grpc_closure on_dns_lookup_done; + /** a closure wrapping on_resolve_address_done, which should be invoked when + the grpc_dns_lookup_ares_locked operation is done. */ + grpc_closure on_dns_lookup_done_locked; /* target name */ const char* name; /* default port to use if none is specified */ const char* default_port; /* pollset_set to be driven by */ grpc_pollset_set* interested_parties; + /* underlying ares_request that the query is performed on */ + grpc_ares_request* ares_request; } grpc_resolve_address_ares_request; -static void on_dns_lookup_done_cb(void* arg, grpc_error* error) { +static void on_dns_lookup_done_locked(void* arg, grpc_error* error) { grpc_resolve_address_ares_request* r = static_cast(arg); + gpr_free(r->ares_request); grpc_resolved_addresses** resolved_addresses = r->addrs_out; if (r->lb_addrs == nullptr || r->lb_addrs->num_addresses == 0) { *resolved_addresses = nullptr; @@ -643,9 +645,9 @@ static void grpc_resolve_address_invoke_dns_lookup_ares_locked( void* arg, grpc_error* unused_error) { grpc_resolve_address_ares_request* r = static_cast(arg); - grpc_dns_lookup_ares_locked( + r->ares_request = grpc_dns_lookup_ares_locked( nullptr /* dns_server */, r->name, r->default_port, r->interested_parties, - &r->on_dns_lookup_done, &r->lb_addrs, false /* check_grpclb */, + &r->on_dns_lookup_done_locked, &r->lb_addrs, false /* check_grpclb */, nullptr /* service_config_json */, r->combiner); } @@ -660,8 +662,8 @@ static void grpc_resolve_address_ares_impl(const char* name, r->combiner = grpc_combiner_create(); r->addrs_out = addrs; r->on_resolve_address_done = on_done; - GRPC_CLOSURE_INIT(&r->on_dns_lookup_done, on_dns_lookup_done_cb, r, - grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&r->on_dns_lookup_done_locked, on_dns_lookup_done_locked, r, + grpc_combiner_scheduler(r->combiner)); r->name = name; r->default_port = default_port; r->interested_parties = interested_parties; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index 1bc457d4cfe..a1231cc4e0d 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -54,7 +54,8 @@ extern void (*grpc_resolve_address_ares)(const char* name, port in \a name. grpc_ares_init() must be called at least once before this function. \a on_done may be called directly in this function without being scheduled with \a exec_ctx, so it must not try to acquire locks that are - being held by the caller. */ + being held by the caller. The returned grpc_ares_request object is owned + by the caller and it is safe to free after on_done is called back. */ extern grpc_ares_request* (*grpc_dns_lookup_ares_locked)( const char* dns_server, const char* name, const char* default_port, grpc_pollset_set* interested_parties, grpc_closure* on_done, @@ -62,7 +63,7 @@ extern grpc_ares_request* (*grpc_dns_lookup_ares_locked)( char** service_config_json, grpc_combiner* combiner); /* Cancel the pending grpc_ares_request \a request */ -void grpc_cancel_ares_request(grpc_ares_request* request); +extern void (*grpc_cancel_ares_request_locked)(grpc_ares_request* request); /* Initialize gRPC ares wrapper. Must be called at least once before grpc_resolve_address_ares(). */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc index d6a76fc8b67..9f293c1ac07 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc @@ -40,7 +40,10 @@ grpc_ares_request* (*grpc_dns_lookup_ares_locked)( grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, grpc_combiner* combiner) = grpc_dns_lookup_ares_locked_impl; -void grpc_cancel_ares_request(grpc_ares_request* r) {} +static void grpc_cancel_ares_request_locked_impl(grpc_ares_request* r) {} + +void (*grpc_cancel_ares_request_locked)(grpc_ares_request* r) = + grpc_cancel_ares_request_locked_impl; grpc_error* grpc_ares_init(void) { return GRPC_ERROR_NONE; } diff --git a/test/core/client_channel/resolvers/dns_resolver_connectivity_test.cc b/test/core/client_channel/resolvers/dns_resolver_connectivity_test.cc index 1c8d0775ab5..eb5a9117484 100644 --- a/test/core/client_channel/resolvers/dns_resolver_connectivity_test.cc +++ b/test/core/client_channel/resolvers/dns_resolver_connectivity_test.cc @@ -82,6 +82,10 @@ static grpc_ares_request* my_dns_lookup_ares_locked( return nullptr; } +static void my_cancel_ares_request_locked(grpc_ares_request* request) { + GPR_ASSERT(request == nullptr); +} + static grpc_core::OrphanablePtr create_resolver( const char* name) { grpc_core::ResolverFactory* factory = @@ -148,6 +152,7 @@ int main(int argc, char** argv) { g_combiner = grpc_combiner_create(); grpc_set_resolver_impl(&test_resolver); grpc_dns_lookup_ares_locked = my_dns_lookup_ares_locked; + grpc_cancel_ares_request_locked = my_cancel_ares_request_locked; grpc_channel_args* result = (grpc_channel_args*)1; { diff --git a/test/core/end2end/fuzzers/api_fuzzer.cc b/test/core/end2end/fuzzers/api_fuzzer.cc index eacfd4a8c3a..2b722d809de 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.cc +++ b/test/core/end2end/fuzzers/api_fuzzer.cc @@ -390,6 +390,10 @@ grpc_ares_request* my_dns_lookup_ares_locked( return nullptr; } +static void my_cancel_ares_request_locked(grpc_ares_request* request) { + GPR_ASSERT(request == nullptr); +} + //////////////////////////////////////////////////////////////////////////////// // client connection @@ -705,6 +709,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { } grpc_set_resolver_impl(&fuzzer_resolver); grpc_dns_lookup_ares_locked = my_dns_lookup_ares_locked; + grpc_cancel_ares_request_locked = my_cancel_ares_request_locked; GPR_ASSERT(g_channel == nullptr); GPR_ASSERT(g_server == nullptr); diff --git a/test/core/end2end/goaway_server_test.cc b/test/core/end2end/goaway_server_test.cc index 8904c3d3253..3f1c5596ad9 100644 --- a/test/core/end2end/goaway_server_test.cc +++ b/test/core/end2end/goaway_server_test.cc @@ -50,6 +50,8 @@ static grpc_ares_request* (*iomgr_dns_lookup_ares_locked)( grpc_lb_addresses** addresses, bool check_grpclb, char** service_config_json, grpc_combiner* combiner); +static void (*iomgr_cancel_ares_request_locked)(grpc_ares_request* request); + static void set_resolve_port(int port) { gpr_mu_lock(&g_mu); g_resolve_port = port; @@ -130,6 +132,12 @@ static grpc_ares_request* my_dns_lookup_ares_locked( return nullptr; } +static void my_cancel_ares_request_locked(grpc_ares_request* request) { + if (request != nullptr) { + iomgr_cancel_ares_request_locked(request); + } +} + int main(int argc, char** argv) { grpc_completion_queue* cq; cq_verifier* cqv; @@ -143,7 +151,9 @@ int main(int argc, char** argv) { default_resolver = grpc_resolve_address_impl; grpc_set_resolver_impl(&test_resolver); iomgr_dns_lookup_ares_locked = grpc_dns_lookup_ares_locked; + iomgr_cancel_ares_request_locked = grpc_cancel_ares_request_locked; grpc_dns_lookup_ares_locked = my_dns_lookup_ares_locked; + grpc_cancel_ares_request_locked = my_cancel_ares_request_locked; int was_cancelled1; int was_cancelled2; From aa3259c10ecd519f71ba65df91c976af4f735733 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 17 Oct 2018 15:42:47 -0700 Subject: [PATCH 49/53] Fix corruption of blocking_resolve_address when ares is used and grpc is restarted --- .../client_channel/resolver/dns/c_ares/dns_resolver_ares.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index dfa52867d8b..c2a4d422038 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -473,7 +473,9 @@ void grpc_resolver_dns_ares_init() { GRPC_LOG_IF_ERROR("ares_library_init() failed", error); return; } - default_resolver = grpc_resolve_address_impl; + if (default_resolver == nullptr) { + default_resolver = grpc_resolve_address_impl; + } grpc_set_resolver_impl(&ares_resolver); grpc_core::ResolverRegistry::Builder::RegisterResolverFactory( grpc_core::UniquePtr( From 1f85f6ba33af282530e28c8e6a7dbaae0da8faed Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Fri, 19 Oct 2018 13:05:35 -0400 Subject: [PATCH 50/53] Do not waste cache lines with unnecessary paddings. Make sure paddings are GPR_CACHELINE_SIZE minus the size of fields in that cache line. Otherwise, we will waste an additional cache line for no good reason. --- src/core/lib/gpr/mpscq.h | 2 +- src/core/lib/iomgr/ev_epoll1_linux.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/gpr/mpscq.h b/src/core/lib/gpr/mpscq.h index 6b67880d1bd..dac2801b4ef 100644 --- a/src/core/lib/gpr/mpscq.h +++ b/src/core/lib/gpr/mpscq.h @@ -40,7 +40,7 @@ typedef struct gpr_mpscq_node { typedef struct gpr_mpscq { gpr_atm head; // make sure head & tail don't share a cacheline - char padding[GPR_CACHELINE_SIZE]; + char padding[GPR_CACHELINE_SIZE - sizeof(head)]; gpr_mpscq_node* tail; gpr_mpscq_node stub; } gpr_mpscq; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 6ef889b0feb..0f58b4a63e7 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -195,7 +195,7 @@ struct grpc_pollset_worker { typedef struct pollset_neighborhood { gpr_mu mu; grpc_pollset* active_root; - char pad[GPR_CACHELINE_SIZE]; + char pad[GPR_CACHELINE_SIZE - sizeof(mu) - sizeof(active_root)]; } pollset_neighborhood; struct grpc_pollset { From 8c1670d5475e50e1efc876f2d45d58574495f891 Mon Sep 17 00:00:00 2001 From: Noah Eisen Date: Fri, 19 Oct 2018 10:23:18 -0700 Subject: [PATCH 51/53] Fix asan --- src/core/lib/channel/channel_trace.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/lib/channel/channel_trace.cc b/src/core/lib/channel/channel_trace.cc index fe81acb617f..f0d21db32a8 100644 --- a/src/core/lib/channel/channel_trace.cc +++ b/src/core/lib/channel/channel_trace.cc @@ -108,16 +108,20 @@ void ChannelTrace::AddTraceEventHelper(TraceEvent* new_trace_event) { } void ChannelTrace::AddTraceEvent(Severity severity, grpc_slice data) { - if (max_event_memory_ == 0) + if (max_event_memory_ == 0) { + grpc_slice_unref_internal(data); return; // tracing is disabled if max_event_memory_ == 0 + } AddTraceEventHelper(New(severity, data)); } void ChannelTrace::AddTraceEventWithReference( Severity severity, grpc_slice data, RefCountedPtr referenced_entity) { - if (max_event_memory_ == 0) + if (max_event_memory_ == 0) { + grpc_slice_unref_internal(data); return; // tracing is disabled if max_event_memory_ == 0 + } // create and fill up the new event AddTraceEventHelper( New(severity, data, std::move(referenced_entity))); From 58a85ffc5e33b2867280906605d94dfcb3c888b5 Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Fri, 19 Oct 2018 14:23:04 -0400 Subject: [PATCH 52/53] Use union to make alignement robust. Suggested-by: vjpai@google.com --- src/core/lib/gpr/mpscq.h | 6 ++++-- src/core/lib/iomgr/ev_epoll1_linux.cc | 10 +++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/core/lib/gpr/mpscq.h b/src/core/lib/gpr/mpscq.h index dac2801b4ef..5ded2522bd6 100644 --- a/src/core/lib/gpr/mpscq.h +++ b/src/core/lib/gpr/mpscq.h @@ -38,9 +38,11 @@ typedef struct gpr_mpscq_node { // Actual queue type typedef struct gpr_mpscq { - gpr_atm head; // make sure head & tail don't share a cacheline - char padding[GPR_CACHELINE_SIZE - sizeof(head)]; + union { + char padding[GPR_CACHELINE_SIZE]; + gpr_atm head; + }; gpr_mpscq_node* tail; gpr_mpscq_node stub; } gpr_mpscq; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 0f58b4a63e7..38571b1957b 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -193,9 +193,13 @@ struct grpc_pollset_worker { #define MAX_NEIGHBORHOODS 1024 typedef struct pollset_neighborhood { - gpr_mu mu; - grpc_pollset* active_root; - char pad[GPR_CACHELINE_SIZE - sizeof(mu) - sizeof(active_root)]; + union { + char pad[GPR_CACHELINE_SIZE]; + struct { + gpr_mu mu; + grpc_pollset* active_root; + }; + }; } pollset_neighborhood; struct grpc_pollset { From ce4bd108ea18891836eb6372da404446ef1a1932 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 15 Oct 2018 12:14:21 -0700 Subject: [PATCH 53/53] Fix CallOpSet copy/assignment to reset cq_tag --- include/grpcpp/impl/codegen/call.h | 13 +++++++++++++ src/cpp/server/server_cc.cc | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/grpcpp/impl/codegen/call.h b/include/grpcpp/impl/codegen/call.h index 7cadea00555..789ea805a3c 100644 --- a/include/grpcpp/impl/codegen/call.h +++ b/include/grpcpp/impl/codegen/call.h @@ -624,6 +624,19 @@ class CallOpSet : public CallOpSetInterface, public Op6 { public: CallOpSet() : cq_tag_(this), return_tag_(this), call_(nullptr) {} + + // The copy constructor and assignment operator reset the value of + // cq_tag_ and return_tag_ since those are only meaningful on a specific + // object, not across objects. + CallOpSet(const CallOpSet& other) + : cq_tag_(this), return_tag_(this), call_(other.call_) {} + CallOpSet& operator=(const CallOpSet& other) { + cq_tag_ = this; + return_tag_ = this; + call_ = other.call_; + return *this; + } + void FillOps(grpc_call* call, grpc_op* ops, size_t* nops) override { this->Op1::AddOp(ops, nops); this->Op2::AddOp(ops, nops); diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 373a9250593..7aeddff6435 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -685,8 +685,8 @@ void Server::PerformOpsOnCall(internal::CallOpSetInterface* ops, size_t nops = 0; grpc_op cops[MAX_OPS]; ops->FillOps(call->call(), cops, &nops); - // TODO(vjpai): Use ops->cq_tag once this case supports callbacks - auto result = grpc_call_start_batch(call->call(), cops, nops, ops, nullptr); + auto result = + grpc_call_start_batch(call->call(), cops, nops, ops->cq_tag(), nullptr); if (result != GRPC_CALL_OK) { gpr_log(GPR_ERROR, "Fatal: grpc_call_start_batch returned %d", result); grpc_call_log_batch(__FILE__, __LINE__, GPR_LOG_SEVERITY_ERROR,