Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Benchmarking, dumping and munging

Browse Source
pull/288/head
Jon Skeet 16 years ago
parent 0c89aa1fdb
commit 8f8186a30b
28 changed files with 6686 additions and 19 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      .gitignore
  2. 10
      ProtocolBuffers.build
  3. 140
      protos/google/protobuf/benchmark.proto
  4. 5742
      src/ProtoBench/BenchmarkProtoFile.cs
  5. 90
      src/ProtoBench/Program.cs
  6. 36
      src/ProtoBench/Properties/AssemblyInfo.cs
  7. 62
      src/ProtoBench/ProtoBench.csproj
  8. 3
      src/ProtoBench/app.config
  9. 43
      src/ProtoDump/Program.cs
  10. 36
      src/ProtoDump/Properties/AssemblyInfo.cs
  11. 59
      src/ProtoDump/ProtoDump.csproj
  12. 3
      src/ProtoDump/app.config
  13. 5
      src/ProtoGen.Test/DependencyResolutionTest.cs
  14. 10
      src/ProtoGen.Test/GeneratorTest.cs
  15. 1
      src/ProtoGen.Test/ProtoGen.Test.csproj
  16. 2
      src/ProtoGen/Program.cs
  17. 228
      src/ProtoMunge/Program.cs
  18. 36
      src/ProtoMunge/Properties/AssemblyInfo.cs
  19. 59
      src/ProtoMunge/ProtoMunge.csproj
  20. 3
      src/ProtoMunge/app.config
  21. 45
      src/ProtocolBuffers.Test/MessageUtilTest.cs
  22. 1
      src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj
  23. 18
      src/ProtocolBuffers.sln
  24. 4
      src/ProtocolBuffers/ByteString.cs
  25. 62
      src/ProtocolBuffers/MessageUtil.cs
  26. 1
      src/ProtocolBuffers/ProtocolBuffers.csproj
  27. BIN
      testdata/benchmark_message1.dat
  28. BIN
      testdata/benchmark_message2.dat

6
.gitignore vendored
Unescape View File

@ -6,10 +6,16 @@ src/ProtocolBuffers/bin/
src/ProtocolBuffers/obj/
src/ProtocolBuffers.Test/bin/
src/ProtocolBuffers.Test/obj/
src/ProtoBench/bin/
src/ProtoBench/obj/
src/ProtoDump/bin/
src/ProtoDump/obj/
src/ProtoGen/bin/
src/ProtoGen/obj/
src/ProtoGen.Test/bin/
src/ProtoGen.Test/obj/
src/ProtoMunge/bin/
src/ProtoMunge/obj/
tmp/
dist/
*.user

10
ProtocolBuffers.build
Unescape View File

@ -72,6 +72,7 @@
workingdir="${tmp-dir}">
<arg value="--proto_path=${protos-dir}" />
<arg value="--descriptor_set_out=compiled.pb" />
<arg file="${protos-dir}/google/protobuf/benchmark.proto" />
<arg file="${protos-dir}/google/protobuf/descriptor.proto" />
<arg file="${protos-dir}/google/protobuf/csharp_options.proto" />
<arg file="${protos-dir}/google/protobuf/unittest.proto" />
@ -114,6 +115,12 @@
<include name="AddressBookProtos.cs" />
</fileset>
</copy>
<copy todir="${src}/ProtoBench">
<fileset basedir="${tmp-dir}">
<include name="BenchmarkProtoFile.cs" />
</fileset>
</copy>
</target>
<target name="build"
@ -147,6 +154,9 @@
<fileset basedir="${src}">
<include name="ProtocolBuffers/bin/${build-configuration}/Google.ProtocolBuffers.dll" />
<include name="ProtoGen/bin/${build-configuration}/ProtoGen.exe" />
<include name="ProtoMunge/bin/${build-configuration}/ProtoMunge.exe" />
<include name="ProtoDump/bin/${build-configuration}/ProtoDump.exe" />
<include name="ProtoBench/bin/${build-configuration}/ProtoBench.exe" />
</fileset>
</copy>

140
protos/google/protobuf/benchmark.proto
Unescape View File

@ -0,0 +1,140 @@
import "google/protobuf/csharp_options.proto";
import "google/protobuf/descriptor.proto";
option (google.protobuf.csharp_file_options).namespace = "Google.ProtocolBuffers.BenchmarkProtos";
option (google.protobuf.csharp_file_options).umbrella_classname = "BenchmarkProtoFile";
package proto2.benchmark.v2_api;
option optimize_for = SPEED;
message Message1 {
required string field1 = 1;
optional string field9 = 9;
optional string field18 = 18;
optional bool field80 = 80 [default=false];
optional bool field81 = 81 [default=true];
required int32 field2 = 2;
required int32 field3 = 3;
optional int32 field280 = 280;
optional int32 field6 = 6 [default=0];
optional int64 field22 = 22;
optional string field4 = 4;
repeated fixed64 field5 = 5;
optional bool field59 = 59 [default=false];
optional string field7 = 7;
optional int32 field16 = 16;
optional int32 field130 = 130 [default=0];
optional bool field12 = 12 [default=true];
optional bool field17 = 17 [default=true];
optional bool field13 = 13 [default=true];
optional bool field14 = 14 [default=true];
optional int32 field104 = 104 [default=0];
optional int32 field100 = 100 [default=0];
optional int32 field101 = 101 [default=0];
optional string field102 = 102;
optional string field103 = 103;
optional int32 field29 = 29 [default=0];
optional bool field30 = 30 [default=false];
optional int32 field60 = 60 [default=-1];
optional int32 field271 = 271 [default=-1];
optional int32 field272 = 272 [default=-1];
optional int32 field150 = 150;
optional int32 field23 = 23 [default=0];
optional bool field24 = 24 [default=false];
optional int32 field25 = 25 [default=0];
optional Message2 field15 = 15;
optional bool field78 = 78;
optional int32 field67 = 67 [default=0];
optional int32 field68 = 68;
optional int32 field128 = 128 [default=0];
optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
optional int32 field131 = 131 [default=0];
}
message Message2 {
optional int32 field1 = 1 [default=0];
optional int32 field2 = 2 [default=0];
optional int32 field3 = 3 [default=0];
optional string field15 = 15;
optional bool field12 = 12 [default=true];
optional int64 field13 = 13;
optional int64 field14 = 14;
optional int32 field16 = 16;
optional int32 field19 = 19 [default=2];
optional bool field20 = 20 [default=true];
optional bool field28 = 28 [default=true];
optional fixed64 field21 = 21;
optional int32 field22 = 22;
optional bool field23 = 23 [ default=false ];
optional bool field206 = 206 [default=false];
optional fixed32 field203 = 203;
optional int32 field204 = 204;
optional string field205 = 205;
optional uint64 field207 = 207;
optional uint64 field300 = 300;
}
message Message3 {
optional string field1 = 1;
optional int64 field3 = 3;
optional int64 field4 = 4;
optional int64 field30 = 30;
optional bool field75 = 75 [default=false];
optional string field6 = 6;
optional bytes field2 = 2;
optional int32 field21 = 21 [default=0];
optional int32 field71 = 71;
optional float field25 = 25;
optional int32 field109 = 109 [default=0];
optional int32 field210 = 210 [default=0];
optional int32 field211 = 211 [default=0];
optional int32 field212 = 212 [default=0];
optional int32 field213 = 213 [default=0];
optional int32 field216 = 216 [default=0];
optional int32 field217 = 217 [default=0];
optional int32 field218 = 218 [default=0];
optional int32 field220 = 220 [default=0];
optional int32 field221 = 221 [default=0];
optional float field222 = 222 [default=0.0];
optional int32 field63 = 63;
repeated group Group1 = 10 {
required float field11 = 11;
optional float field26 = 26;
optional string field12 = 12;
optional string field13 = 13;
repeated string field14 = 14;
required uint64 field15 = 15;
optional int32 field5 = 5;
optional string field27 = 27;
optional int32 field28 = 28;
optional string field29 = 29;
optional string field16 = 16;
repeated string field22 = 22;
repeated int32 field73 = 73;
optional int32 field20 = 20 [default=0];
optional string field24 = 24;
optional Message4 field31 = 31;
}
repeated string field128 = 128;
optional int64 field131 = 131;
repeated string field127 = 127;
optional int32 field129 = 129;
repeated int64 field130 = 130;
optional bool field205 = 205 [default=false];
optional bool field206 = 206 [default=false];
}
message Message4 {
optional float field1 = 1;
optional float field2 = 2;
optional float field3 = 3 [default=0.0];
optional bool field4 = 4;
optional bool field5 = 5;
optional bool field6 = 6 [default=true];
optional bool field7 = 7 [default=false];
optional float field8 = 8;
optional bool field9 = 9;
optional float field10 = 10;
optional int64 field11 = 11;
}

5742
src/ProtoBench/BenchmarkProtoFile.cs
View File

File diff suppressed because it is too large Load Diff

90
src/ProtoBench/Program.cs
Unescape View File

@ -0,0 +1,90 @@
using System;
using System.Diagnostics;
using System.IO;
using Google.ProtocolBuffers;
namespace ProtoBench {
/// <summary>
/// Simple benchmarking of arbitrary messages.
/// </summary>
public sealed class Program {
private static readonly TimeSpan MinSampleTime = TimeSpan.FromSeconds(2);
private static readonly TimeSpan TargetTime = TimeSpan.FromSeconds(30);
// Avoid a .NET 3.5 dependency
delegate void Action();
public static int Main(string[] args) {
if (args.Length != 2) {
Console.Error.WriteLine("Usage: ProtoBecnh <descriptor type name> <input data>");
Console.Error.WriteLine("The descriptor type name is the fully-qualified message name,");
Console.Error.WriteLine("including assembly - e.g. Google.ProtocolBuffers.BenchmarkProtos.Message1,ProtoBench");
return 1;
}
IMessage defaultMessage;
try {
defaultMessage = MessageUtil.GetDefaultMessage(args[0]);
} catch (ArgumentException e) {
Console.Error.WriteLine(e.Message);
return 1;
}
try {
IBuilder builder = defaultMessage.WeakCreateBuilderForType();
byte[] inputData = File.ReadAllBytes(args[1]);
ByteString inputString = ByteString.CopyFrom(inputData);
IMessage sampleMessage = defaultMessage.WeakCreateBuilderForType().WeakMergeFrom(inputString).WeakBuild();
sampleMessage.ToByteString();
Console.WriteLine("Benchmarking {0} with file {1}", sampleMessage.GetType().Name, args[1]);
Benchmark("Serialize to byte string", inputData.Length, () => sampleMessage.ToByteString());
Benchmark("Serialize to byte array", inputData.Length, () => sampleMessage.ToByteArray());
Benchmark("Serialize to memory stream", inputData.Length, () => sampleMessage.WriteTo(new MemoryStream()));
Benchmark("Deserialize from byte array", inputData.Length, () =>
defaultMessage.WeakCreateBuilderForType()
.WeakMergeFrom(CodedInputStream.CreateInstance(inputData))
.WeakBuild()
);
Benchmark("Deserialize from byte array", inputData.Length, () =>
defaultMessage.WeakCreateBuilderForType()
.WeakMergeFrom(inputString)
.WeakBuild()
);
return 0;
} catch (Exception e) {
Console.Error.WriteLine("Error: {0}", e.Message);
Console.Error.WriteLine();
Console.Error.WriteLine("Detailed exception information: {0}", e);
return 1;
}
}
private static void Benchmark(string name, int dataSize, Action action) {
// Make sure it's JITted
action();
// Run it progressively more times until we've got a reasonable sample
int iterations = 1;
TimeSpan elapsed = TimeAction(action, iterations);
while (elapsed < MinSampleTime) {
iterations *= 2;
elapsed = TimeAction(action, iterations);
}
// Upscale the sample to the target time. Do this in floating point arithmetic
// to avoid overflow issues.
iterations = (int) ((TargetTime.Ticks / (double)elapsed.Ticks) * iterations);
elapsed = TimeAction(action, iterations);
Console.WriteLine("{0}: {1} iterations in {2:f3}s; {3:f3}MB/s",
name, iterations, elapsed.TotalSeconds,
(iterations * dataSize) / (elapsed.TotalSeconds * 1024 * 1024));
}
private static TimeSpan TimeAction(Action action, int iterations) {
Stopwatch sw = Stopwatch.StartNew();
for (int i = 0; i < iterations; i++) {
action();
}
sw.Stop();
return sw.Elapsed;
}
}
}

36
src/ProtoBench/Properties/AssemblyInfo.cs
Unescape View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("ProtoBench")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("ProtoBench")]
[assembly: AssemblyCopyright("Copyright © 2009")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("0f515d09-9a6c-49ec-8500-14a5303ebadf")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

62
src/ProtoBench/ProtoBench.csproj
Unescape View File

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="3.5" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProductVersion>9.0.30729</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{C7A4A435-2813-41C8-AA87-BD914BA5223D}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Google.ProtocolBuffers.ProtoBench</RootNamespace>
<AssemblyName>ProtoBench</AssemblyName>
<TargetFrameworkVersion>v2.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkSubset>
</TargetFrameworkSubset>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="BenchmarkProtoFile.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ProtocolBuffers\ProtocolBuffers.csproj">
<Project>{6908BDCE-D925-43F3-94AC-A531E6DF2591}</Project>
<Name>ProtocolBuffers</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

3
src/ProtoBench/app.config
Unescape View File

@ -0,0 +1,3 @@
<?xml version="1.0"?>
<configuration>
<startup><supportedRuntime version="v2.0.50727"/></startup></configuration>

43
src/ProtoDump/Program.cs
Unescape View File

@ -0,0 +1,43 @@
using System;
using System.IO;
using System.Reflection;
using Google.ProtocolBuffers;
namespace ProtoDump {
/// <summary>
/// Small utility to load a binary message and dump it in text form
/// </summary>
class Program {
static int Main(string[] args) {
if (args.Length != 2) {
Console.Error.WriteLine("Usage: ProtoDump <descriptor type name> <input data>");
Console.Error.WriteLine("The descriptor type name is the fully-qualified message name,");
Console.Error.WriteLine("including assembly e.g. ProjectNamespace.Message,Company.Project");
return 1;
}
IMessage defaultMessage;
try {
defaultMessage = MessageUtil.GetDefaultMessage(args[0]);
} catch (ArgumentException e) {
Console.Error.WriteLine(e.Message);
return 1;
}
try {
IBuilder builder = defaultMessage.WeakCreateBuilderForType();
if (builder == null) {
Console.Error.WriteLine("Unable to create builder");
return 1;
}
byte[] inputData = File.ReadAllBytes(args[1]);
builder.WeakMergeFrom(ByteString.CopyFrom(inputData));
Console.WriteLine(TextFormat.PrintToString(builder.WeakBuild()));
return 0;
} catch (Exception e) {
Console.Error.WriteLine("Error: {0}", e.Message);
Console.Error.WriteLine();
Console.Error.WriteLine("Detailed exception information: {0}", e);
return 1;
}
}
}
}

36
src/ProtoDump/Properties/AssemblyInfo.cs
Unescape View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("ProtoDump")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("ProtoDump")]
[assembly: AssemblyCopyright("Copyright © 2009")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("fed7572b-d747-4704-a6da-6c3c61088346")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

59
src/ProtoDump/ProtoDump.csproj
Unescape View File

@ -0,0 +1,59 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="3.5" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProductVersion>9.0.30729</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{D7282E99-2DC3-405B-946F-177DB2FD2AE2}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Google.ProtocolBuffers.ProtoDump</RootNamespace>
<AssemblyName>ProtoDump</AssemblyName>
<TargetFrameworkVersion>v2.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkSubset>
</TargetFrameworkSubset>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ProtocolBuffers\ProtocolBuffers.csproj">
<Project>{6908BDCE-D925-43F3-94AC-A531E6DF2591}</Project>
<Name>ProtocolBuffers</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

3
src/ProtoDump/app.config
Unescape View File

@ -0,0 +1,3 @@
<?xml version="1.0"?>
<configuration>
<startup><supportedRuntime version="v2.0.50727"/></startup></configuration>

5
src/ProtoGen.Test/DependencyResolutionTest.cs
Unescape View File

@ -1,10 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text;
using Google.ProtocolBuffers.DescriptorProtos;
using Google.ProtocolBuffers.Descriptors;
using NUnit.Framework;
using Google.ProtocolBuffers.DescriptorProtos;
using Google.ProtocolBuffers.ProtoGen;
namespace Google.ProtocolBuffers.ProtoGen {
/// <summary>

10
src/ProtoGen.Test/GeneratorTest.cs
Unescape View File

@ -1,10 +0,0 @@
using Google.ProtocolBuffers.DescriptorProtos;
using NUnit.Framework;
using Google.ProtocolBuffers.Descriptors;
namespace Google.ProtocolBuffers.ProtoGen {
[TestFixture]
public class GeneratorTest {
}
}

1
src/ProtoGen.Test/ProtoGen.Test.csproj
Unescape View File

@ -45,7 +45,6 @@
</ItemGroup>
<ItemGroup>
<Compile Include="DependencyResolutionTest.cs" />
<Compile Include="GeneratorTest.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>

2
src/ProtoGen/Program.cs
Unescape View File

@ -23,8 +23,6 @@ namespace Google.ProtocolBuffers.ProtoGen {
Generator generator = Generator.CreateGenerator(options);
generator.Generate();
return 0;
} catch (Exception e) {
Console.Error.WriteLine("Error: {0}", e.Message);

228
src/ProtoMunge/Program.cs
Unescape View File

@ -0,0 +1,228 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using Google.ProtocolBuffers.Descriptors;
namespace Google.ProtocolBuffers.ProtoMunge
{
/// <summary>
/// Utility console application which takes a message descriptor and a corresponding message,
/// and produces a new message with similar but random data. The data is the same length
/// as the original, but with random values within appropriate bands. (For instance, a compressed
/// integer in the range 0-127 will end up as another integer in the same range, to keep the length
/// the same.)
/// TODO(jonskeet): Potentially refactor to use an instance instead, making it simpler to
/// be thread-safe for external use.
/// </summary>
public sealed class Program {
static readonly Random rng = new Random();
static int Main(string[] args) {
if (args.Length != 3) {
Console.Error.WriteLine("Usage: ProtoMunge <descriptor type name> <input data> <output file>");
Console.Error.WriteLine("The descriptor type name is the fully-qualified message name, including assembly.");
Console.Error.WriteLine("(At a future date it may be possible to do this without building the .NET assembly at all.)");
return 1;
}
IMessage defaultMessage;
try {
defaultMessage = MessageUtil.GetDefaultMessage(args[0]);
} catch (ArgumentException e) {
Console.Error.WriteLine(e.Message);
return 1;
}
try {
IBuilder builder = defaultMessage.WeakCreateBuilderForType();
byte[] inputData = File.ReadAllBytes(args[1]);
builder.WeakMergeFrom(ByteString.CopyFrom(inputData));
IMessage original = builder.WeakBuild();
IMessage munged = Munge(original);
if (original.SerializedSize != munged.SerializedSize) {
throw new Exception("Serialized sizes don't match");
}
File.WriteAllBytes(args[2], munged.ToByteArray());
return 0;
} catch (Exception e) {
Console.Error.WriteLine("Error: {0}", e.Message);
Console.Error.WriteLine();
Console.Error.WriteLine("Detailed exception information: {0}", e);
return 1;
}
}
/// <summary>
/// Munges a message recursively.
/// </summary>
/// <returns>A new message of the same type as the original message,
/// but munged so that all the data is desensitised.</returns>
private static IMessage Munge(IMessage message) {
IBuilder builder = message.WeakCreateBuilderForType();
foreach (var pair in message.AllFields) {
if (pair.Key.IsRepeated) {
foreach (object singleValue in (IEnumerable)pair.Value) {
builder.WeakAddRepeatedField(pair.Key, CheckedMungeValue(pair.Key, singleValue));
}
} else {
builder[pair.Key] = CheckedMungeValue(pair.Key, pair.Value);
}
}
IMessage munged = builder.WeakBuild();
if (message.SerializedSize != munged.SerializedSize) {
Console.WriteLine("Sub message sizes: {0}/{1}", message.SerializedSize, munged.SerializedSize);
}
return munged;
}
/// <summary>
/// Munges a single value and checks that the length ends up the same as it was before.
/// </summary>
private static object CheckedMungeValue(FieldDescriptor fieldDescriptor, object value) {
int currentSize = CodedOutputStream.ComputeFieldSize(fieldDescriptor.FieldType, fieldDescriptor.FieldNumber, value);
object mungedValue = MungeValue(fieldDescriptor, value);
int mungedSize = CodedOutputStream.ComputeFieldSize(fieldDescriptor.FieldType, fieldDescriptor.FieldNumber, mungedValue);
// Exceptions log more easily than assertions
if (currentSize != mungedSize) {
throw new Exception("Munged value had wrong size. Field type: " + fieldDescriptor.FieldType
+ "; old value: " + value + "; new value: " + mungedValue);
}
return mungedValue;
}
/// <summary>
/// Munges a single value of the specified field descriptor. (i.e. if the field is
/// actually a repeated int, this method receives a single int value to munge, and
/// is called multiple times).
/// </summary>
private static object MungeValue(FieldDescriptor fieldDescriptor, object value) {
switch (fieldDescriptor.FieldType) {
case FieldType.SInt64:
case FieldType.Int64:
return (long) MungeVarint64((ulong) (long)value);
case FieldType.UInt64:
return MungeVarint64((ulong)value);
case FieldType.SInt32:
return (int)MungeVarint32((uint)(int)value);
case FieldType.Int32:
return MungeInt32((int) value);
case FieldType.UInt32:
return MungeVarint32((uint)value);
case FieldType.Double:
return rng.NextDouble();
case FieldType.Float:
return (float)rng.NextDouble();
case FieldType.Fixed64: {
byte[] data = new byte[8];
rng.NextBytes(data);
return BitConverter.ToUInt64(data, 0);
}
case FieldType.Fixed32: {
byte[] data = new byte[4];
rng.NextBytes(data);
return BitConverter.ToUInt32(data, 0);
}
case FieldType.Bool:
return rng.Next(2) == 1;
case FieldType.String:
return MungeString((string)value);
case FieldType.Group:
case FieldType.Message:
return Munge((IMessage)value);
case FieldType.Bytes:
return MungeByteString((ByteString)value);
case FieldType.SFixed64: {
byte[] data = new byte[8];
rng.NextBytes(data);
return BitConverter.ToInt64(data, 0);
}
case FieldType.SFixed32: {
byte[] data = new byte[4];
rng.NextBytes(data);
return BitConverter.ToInt32(data, 0);
}
case FieldType.Enum:
return MungeEnum(fieldDescriptor, (int) value);
default:
// TODO(jonskeet): Different exception?
throw new ArgumentException("Invalid field descriptor");
}
}
private static object MungeString(string original) {
foreach (char c in original) {
if (c > 127) {
throw new ArgumentException("Can't handle non-ascii yet");
}
}
char[] chars = new char[original.Length];
// Convert to pure ASCII - no control characters.
for (int i = 0; i < chars.Length; i++) {
chars[i] = (char) rng.Next(32, 127);
}
return new string(chars);
}
/// <summary>
/// Int32 fields are slightly strange - we need to keep the sign the same way it is:
/// negative numbers can munge to any other negative number (it'll always take
/// 10 bytes) but positive numbers have to stay positive, so we can't use the
/// full range of 32 bits.
/// </summary>
private static int MungeInt32(int value) {
if (value < 0) {
return rng.Next(int.MinValue, 0);
}
int length = CodedOutputStream.ComputeRawVarint32Size((uint) value);
uint min = length == 1 ? 0 : 1U << ((length - 1) * 7);
uint max = length == 5 ? int.MaxValue : (1U << (length * 7)) - 1;
return (int) NextRandomUInt64(min, max);
}
private static uint MungeVarint32(uint original) {
int length = CodedOutputStream.ComputeRawVarint32Size(original);
uint min = length == 1 ? 0 : 1U << ((length - 1) * 7);
uint max = length == 5 ? uint.MaxValue : (1U << (length * 7)) - 1;
return (uint)NextRandomUInt64(min, max);
}
private static ulong MungeVarint64(ulong original) {
int length = CodedOutputStream.ComputeRawVarint64Size(original);
ulong min = length == 1 ? 0 : 1UL << ((length - 1) * 7);
ulong max = length == 10 ? ulong.MaxValue : (1UL<< (length * 7)) - 1;
return NextRandomUInt64(min, max);
}
/// <summary>
/// Returns a random number in the range [min, max] (both inclusive).
/// </summary>
private static ulong NextRandomUInt64(ulong min, ulong max) {
if (min > max) {
throw new ArgumentException("min must be <= max; min=" + min + "; max = " + max);
}
ulong range = max - min;
// This isn't actually terribly good at very large ranges - but it doesn't really matter for the sake
// of this program.
return min + (ulong)(range * rng.NextDouble());
}
private static object MungeEnum(FieldDescriptor fieldDescriptor, int original) {
// Find all the values which get encoded to the same size as the current value, and pick one at random
int originalSize = CodedOutputStream.ComputeRawVarint32Size((uint)original);
List<EnumValueDescriptor> sameSizeValues = new List<EnumValueDescriptor> ();
foreach (EnumValueDescriptor candidate in fieldDescriptor.EnumType.Values) {
if (CodedOutputStream.ComputeRawVarint32Size((uint)candidate.Number) == originalSize) {
sameSizeValues.Add(candidate);
}
}
return sameSizeValues[rng.Next(sameSizeValues.Count)].Number;
}
private static object MungeByteString(ByteString byteString) {
byte[] data = new byte[byteString.Length];
rng.NextBytes(data);
return ByteString.CopyFrom(data);
}
}
}

36
src/ProtoMunge/Properties/AssemblyInfo.cs
Unescape View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("ProtoMunge")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("ProtoMunge")]
[assembly: AssemblyCopyright("Copyright © 2008")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("4d26ed0e-a6ca-4df9-bb87-59429d49b676")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

59
src/ProtoMunge/ProtoMunge.csproj
Unescape View File

@ -0,0 +1,59 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="3.5" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProductVersion>9.0.30729</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{8F09AF72-3327-4FA7-BC09-070B80221AB9}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Google.ProtocolBuffers.ProtoMunge</RootNamespace>
<AssemblyName>ProtoMunge</AssemblyName>
<TargetFrameworkVersion>v2.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkSubset>
</TargetFrameworkSubset>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ProtocolBuffers\ProtocolBuffers.csproj">
<Project>{6908BDCE-D925-43F3-94AC-A531E6DF2591}</Project>
<Name>ProtocolBuffers</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

3
src/ProtoMunge/app.config
Unescape View File

@ -0,0 +1,3 @@
<?xml version="1.0"?>
<configuration>
<startup><supportedRuntime version="v2.0.50727"/></startup></configuration>

45
src/ProtocolBuffers.Test/MessageUtilTest.cs
Unescape View File

@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using System.Text;
using Google.ProtocolBuffers.TestProtos;
using NUnit.Framework;
namespace Google.ProtocolBuffers {
[TestFixture]
public class MessageUtilTest {
[Test]
[ExpectedException(typeof(ArgumentNullException))]
public void NullTypeName() {
MessageUtil.GetDefaultMessage((string)null);
}
[Test]
[ExpectedException(typeof(ArgumentException))]
public void InvalidTypeName() {
MessageUtil.GetDefaultMessage("invalidtypename");
}
[Test]
public void ValidTypeName() {
Assert.AreSame(TestAllTypes.DefaultInstance, MessageUtil.GetDefaultMessage(typeof(TestAllTypes).AssemblyQualifiedName));
}
[Test]
[ExpectedException(typeof(ArgumentNullException))]
public void NullType() {
MessageUtil.GetDefaultMessage((Type)null);
}
[Test]
[ExpectedException(typeof(ArgumentException))]
public void NonMessageType() {
MessageUtil.GetDefaultMessage(typeof(string));
}
[Test]
public void ValidType() {
Assert.AreSame(TestAllTypes.DefaultInstance, MessageUtil.GetDefaultMessage(typeof(TestAllTypes)));
}
}
}

1
src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj
Unescape View File

@ -58,6 +58,7 @@
<Compile Include="MessageStreamIteratorTest.cs" />
<Compile Include="MessageStreamWriterTest.cs" />
<Compile Include="MessageTest.cs" />
<Compile Include="MessageUtilTest.cs" />
<Compile Include="NameHelpersTest.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="ReflectionTester.cs" />

18
src/ProtocolBuffers.sln
Unescape View File

@ -11,6 +11,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProtoGen.Test", "ProtoGen.T
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AddressBook", "AddressBook\AddressBook.csproj", "{A31F5FB2-4FF3-432A-B35B-5CD203606311}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProtoMunge", "ProtoMunge\ProtoMunge.csproj", "{8F09AF72-3327-4FA7-BC09-070B80221AB9}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProtoBench", "ProtoBench\ProtoBench.csproj", "{C7A4A435-2813-41C8-AA87-BD914BA5223D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProtoDump", "ProtoDump\ProtoDump.csproj", "{D7282E99-2DC3-405B-946F-177DB2FD2AE2}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -37,6 +43,18 @@ Global
{A31F5FB2-4FF3-432A-B35B-5CD203606311}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A31F5FB2-4FF3-432A-B35B-5CD203606311}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A31F5FB2-4FF3-432A-B35B-5CD203606311}.Release|Any CPU.Build.0 = Release|Any CPU
{8F09AF72-3327-4FA7-BC09-070B80221AB9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8F09AF72-3327-4FA7-BC09-070B80221AB9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8F09AF72-3327-4FA7-BC09-070B80221AB9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8F09AF72-3327-4FA7-BC09-070B80221AB9}.Release|Any CPU.Build.0 = Release|Any CPU
{C7A4A435-2813-41C8-AA87-BD914BA5223D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C7A4A435-2813-41C8-AA87-BD914BA5223D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C7A4A435-2813-41C8-AA87-BD914BA5223D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C7A4A435-2813-41C8-AA87-BD914BA5223D}.Release|Any CPU.Build.0 = Release|Any CPU
{D7282E99-2DC3-405B-946F-177DB2FD2AE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D7282E99-2DC3-405B-946F-177DB2FD2AE2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D7282E99-2DC3-405B-946F-177DB2FD2AE2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D7282E99-2DC3-405B-946F-177DB2FD2AE2}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

4
src/ProtocolBuffers/ByteString.cs
Unescape View File

@ -184,7 +184,7 @@ namespace Google.ProtocolBuffers {
output = CodedOutputStream.CreateInstance(buffer);
}
public ByteString Build() {
internal ByteString Build() {
output.CheckNoSpaceLeft();
// We can be confident that the CodedOutputStream will not modify the
@ -193,7 +193,7 @@ namespace Google.ProtocolBuffers {
return new ByteString(buffer);
}
public CodedOutputStream CodedOutput {
internal CodedOutputStream CodedOutput {
get {
return output;
}

62
src/ProtocolBuffers/MessageUtil.cs
Unescape View File

@ -0,0 +1,62 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Text;
namespace Google.ProtocolBuffers {
/// <summary>
/// Utilities for arbitrary messages of an unknown type. This class does not use
/// generics precisely because it is designed for dynamically discovered types.
/// </summary>
public static class MessageUtil {
/// <summary>
/// Returns the default message for the given type. If an exception is thrown
/// (directly from this code), the message will be suitable to be displayed to a user.
/// </summary>
/// <param name="type"></param>
/// <exception cref="ArgumentNullException">The type parameter is null.</exception>
/// <exception cref="ArgumentException">The type doesn't implement IMessage, or doesn't
/// have a static DefaultMessage property of the same type, or is generic or abstract.</exception>
/// <returns></returns>
public static IMessage GetDefaultMessage(Type type) {
if (type == null) {
throw new ArgumentNullException("type", "No type specified.");
}
if (type.IsAbstract || type.IsGenericTypeDefinition) {
throw new ArgumentException("Unable to get a default message for an abstract or generic type (" + type.FullName + ")");
}
if (!typeof(IMessage).IsAssignableFrom(type)) {
throw new ArgumentException("Unable to get a default message for non-message type (" + type.FullName + ")");
}
PropertyInfo property = type.GetProperty("DefaultInstance", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic);
if (property == null) {
throw new ArgumentException(type.FullName + " doesn't have a static DefaultInstance property");
}
if (property.PropertyType != type) {
throw new ArgumentException(type.FullName + ".DefaultInstance property is of the wrong type");
}
return (IMessage) property.GetValue(null, null);
}
/// <summary>
/// Returns the default message for the type with the given name. This is just
/// a convenience wrapper around calling Type.GetType and then GetDefaultMessage.
/// If an exception is thrown, the message will be suitable to be displayed to a user.
/// </summary>
/// <param name="typeName"></param>
/// <exception cref="ArgumentNullException">The typeName parameter is null.</exception>
/// <exception cref="ArgumentException">The type doesn't implement IMessage, or doesn't
/// have a static DefaultMessage property of the same type, or can't be found.</exception>
public static IMessage GetDefaultMessage(string typeName) {
if (typeName == null) {
throw new ArgumentNullException("typeName", "No type name specified.");
}
Type type = Type.GetType(typeName);
if (type == null) {
throw new ArgumentException("Unable to load type " + typeName);
}
return GetDefaultMessage(type);
}
}
}

1
src/ProtocolBuffers/ProtocolBuffers.csproj
Unescape View File

@ -97,6 +97,7 @@
<Compile Include="IService.cs" />
<Compile Include="MessageStreamIterator.cs" />
<Compile Include="MessageStreamWriter.cs" />
<Compile Include="MessageUtil.cs" />
<Compile Include="NameHelpers.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="RpcUtil.cs" />

BIN
testdata/benchmark_message1.dat vendored
View File

Binary file not shown.

BIN
testdata/benchmark_message2.dat vendored
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save