Initial benchmarking.

benchmarks/ProtoBench.java

@@ -0,0 +1,145 @@
+package com.google.protocolbuffers;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.lang.reflect.Method;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.Message;
+
+public class ProtoBench {
+  
+  private static long MIN_SAMPLE_TIME_MS = 2 * 1000;
+  private static long TARGET_TIME_MS = 30 * 1000;
+
+  private ProtoBench() {
+    // Prevent instantiation
+  }
+  
+  public static void main(String[] args) {
+    if (args.length < 2 || (args.length % 2) != 0) {
+      System.err.println("Usage: ProtoBench <descriptor type name> <input data>");
+      System.err.println("The descriptor type name is the fully-qualified message name,");
+      System.err.println("e.g. com.google.protocolbuffers.benchmark.Message1");
+      System.err.println("(You can specify multiple pairs of descriptor type name and input data.)");
+      System.exit(1);
+    }
+    boolean success = true;
+    for (int i = 0; i < args.length; i += 2) {
+      success &= runTest(args[i], args[i + 1]);
+    }
+    System.exit(success ? 0 : 1);
+  }
+
+  /**
+   * Runs a single test. Error messages are displayed to stderr, and the return value
+   * indicates general success/failure.
+   */
+  public static boolean runTest(String type, String file) {
+    System.out.println("Benchmarking " + type + " with file " + file);
+    final Message defaultMessage;
+    try {
+      Class<?> clazz = Class.forName(type);
+      Method method = clazz.getDeclaredMethod("getDefaultInstance");
+      defaultMessage = (Message) method.invoke(null);
+    } catch (Exception e) {
+      // We want to do the same thing with all exceptions. Not generally nice,
+      // but this is slightly different.
+      System.err.println("Unable to get default message for " + type);
+      return false;
+    }
+    
+    try {
+      final byte[] inputData = readAllBytes(file);
+      final ByteArrayInputStream inputStream = new ByteArrayInputStream(inputData);
+      final ByteString inputString = ByteString.copyFrom(inputData);
+      final Message sampleMessage = defaultMessage.newBuilderForType().mergeFrom(inputString).build();
+      benchmark("Serialize to byte string", inputData.length, new Action() {
+        public void execute() { sampleMessage.toByteString(); }
+      });      
+      benchmark("Serialize to byte array", inputData.length, new Action() {
+        public void execute() { sampleMessage.toByteArray(); }
+      });
+      benchmark("Serialize to memory stream", inputData.length, new Action() {
+        public void execute() throws IOException { 
+          sampleMessage.writeTo(new ByteArrayOutputStream()); 
+        }
+      });
+      benchmark("Deserialize from byte string", inputData.length, new Action() {
+        public void execute() throws IOException { 
+          defaultMessage.newBuilderForType().mergeFrom(inputString).build();
+        }
+      });
+      benchmark("Deserialize from byte array", inputData.length, new Action() {
+        public void execute() throws IOException { 
+          defaultMessage.newBuilderForType()
+            .mergeFrom(CodedInputStream.newInstance(inputData)).build();
+        }
+      });
+      benchmark("Deserialize from memory stream", inputData.length, new Action() {
+        public void execute() throws IOException { 
+          defaultMessage.newBuilderForType()
+            .mergeFrom(CodedInputStream.newInstance(inputStream)).build();
+          inputStream.reset();
+        }
+      });
+      System.out.println();
+      return true;
+    } catch (Exception e) {
+      System.err.println("Error: " + e.getMessage());
+      System.err.println("Detailed exception information:");
+      e.printStackTrace(System.err);
+      return false;
+    }
+  }
+  
+  private static void benchmark(String name, long dataSize, Action action) throws IOException {
+    // Make sure it's JITted "reasonably" hard before running the first progress test
+    for (int i=0; i < 100; i++) {
+      action.execute();
+    }
+    
+    // Run it progressively more times until we've got a reasonable sample
+    int iterations = 1;
+    long elapsed = timeAction(action, iterations);
+    while (elapsed < MIN_SAMPLE_TIME_MS) {
+      iterations *= 2;
+      elapsed = timeAction(action, iterations);
+    }
+    // Upscale the sample to the target time. Do this in floating point arithmetic
+    // to avoid overflow issues.
+    iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations);
+    elapsed = timeAction(action, iterations);
+    System.out.println(name + ": " + iterations + " iterations in "
+         + (elapsed/1000f) + "s; " 
+         + (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f) 
+         + "MB/s");
+  }
+  
+  private static long timeAction(Action action, int iterations) throws IOException {
+    long start = System.currentTimeMillis();
+    for (int i = 0; i < iterations; i++) {
+      action.execute();
+    }
+    long end = System.currentTimeMillis();
+    return end - start;
+  }
+  
+  private static byte[] readAllBytes(String filename) throws IOException {
+    RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
+    byte[] content = new byte[(int) file.length()];
+    file.readFully(content);
+    return content;
+  }
+
+  /**
+   * Interface used to capture a single action to benchmark.
+   */
+  interface Action {
+    void execute() throws IOException;
+  }
+}

benchmarks/google_size.proto

@@ -0,0 +1,136 @@
+package benchmarks;
+
+option java_outer_classname = "GoogleSize";
+option optimize_for = CODE_SIZE;
+
+message SizeMessage1 {
+  required string field1 = 1;
+  optional string field9 = 9;
+  optional string field18 = 18;
+  optional bool field80 = 80 [default=false];
+  optional bool field81 = 81 [default=true];
+  required int32 field2 = 2;
+  required int32 field3 = 3;
+  optional int32 field280 = 280;
+  optional int32 field6 = 6 [default=0];
+  optional int64 field22 = 22;
+  optional string field4 = 4;
+  repeated fixed64 field5 = 5;
+  optional bool field59 = 59 [default=false];
+  optional string field7 = 7;
+  optional int32 field16 = 16;
+  optional int32 field130 = 130 [default=0];
+  optional bool field12 = 12 [default=true];
+  optional bool field17 = 17 [default=true];
+  optional bool field13 = 13 [default=true];
+  optional bool field14 = 14 [default=true];
+  optional int32 field104 = 104 [default=0];
+  optional int32 field100 = 100 [default=0];
+  optional int32 field101 = 101 [default=0];
+  optional string field102 = 102;
+  optional string field103 = 103;
+  optional int32 field29 = 29 [default=0];
+  optional bool field30 = 30 [default=false];
+  optional int32 field60 = 60 [default=-1];
+  optional int32 field271 = 271 [default=-1];
+  optional int32 field272 = 272 [default=-1];
+  optional int32 field150 = 150;
+  optional int32 field23 = 23 [default=0];
+  optional bool field24 = 24 [default=false];
+  optional int32 field25 = 25 [default=0];
+  optional SizeMessage1SubMessage field15 = 15;
+  optional bool field78 = 78;
+  optional int32 field67 = 67 [default=0];
+  optional int32 field68 = 68;
+  optional int32 field128 = 128 [default=0];
+  optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
+  optional int32 field131 = 131 [default=0];
+}
+
+message SizeMessage1SubMessage {
+  optional int32 field1 = 1 [default=0];
+  optional int32 field2 = 2 [default=0];
+  optional int32 field3 = 3 [default=0];
+  optional string field15 = 15;
+  optional bool field12 = 12 [default=true];
+  optional int64 field13 = 13;
+  optional int64 field14 = 14;
+  optional int32 field16 = 16;
+  optional int32 field19 = 19 [default=2];
+  optional bool field20  = 20 [default=true];
+  optional bool field28 = 28 [default=true];
+  optional fixed64 field21 = 21;
+  optional int32 field22 = 22;
+  optional bool field23 = 23 [ default=false ];
+  optional bool field206 = 206 [default=false];
+  optional fixed32 field203 = 203;
+  optional int32 field204 = 204;
+  optional string field205 = 205;
+  optional uint64 field207 = 207;
+  optional uint64 field300 = 300;
+}
+
+message SizeMessage2 {
+  optional string field1 = 1;
+  optional int64 field3 = 3;
+  optional int64 field4 = 4;
+  optional int64 field30 = 30;
+  optional bool field75  = 75 [default=false];
+  optional string field6 = 6;
+  optional bytes field2 = 2;
+  optional int32 field21 = 21 [default=0];
+  optional int32 field71 = 71;
+  optional float field25 = 25;
+  optional int32 field109 = 109 [default=0];
+  optional int32 field210 = 210 [default=0];
+  optional int32 field211 = 211 [default=0];
+  optional int32 field212 = 212 [default=0];
+  optional int32 field213 = 213 [default=0];
+  optional int32 field216 = 216 [default=0];
+  optional int32 field217 = 217 [default=0];
+  optional int32 field218 = 218 [default=0];
+  optional int32 field220 = 220 [default=0];
+  optional int32 field221 = 221 [default=0];
+  optional float field222 = 222 [default=0.0];
+  optional int32 field63 = 63;
+
+  repeated group Group1 = 10 {
+    required float field11 = 11;
+    optional float field26 = 26;
+    optional string field12 = 12;
+    optional string field13 = 13;
+    repeated string field14 = 14;
+    required uint64 field15 = 15;
+    optional int32 field5 = 5;
+    optional string field27 = 27;
+    optional int32 field28 = 28;
+    optional string field29 = 29;
+    optional string field16 = 16;
+    repeated string field22 = 22;
+    repeated int32 field73 = 73;
+    optional int32 field20 = 20 [default=0];
+    optional string field24 = 24;
+    optional SizeMessage2GroupedMessage field31 = 31;
+  }
+  repeated string field128 = 128;
+  optional int64 field131 = 131;
+  repeated string field127 = 127;
+  optional int32 field129 = 129;
+  repeated int64 field130 = 130;
+  optional bool field205 = 205 [default=false];
+  optional bool field206 = 206 [default=false];
+}
+
+message SizeMessage2GroupedMessage {
+  optional float field1 = 1;
+  optional float field2 = 2;
+  optional float field3 = 3 [default=0.0];
+  optional bool field4 = 4;
+  optional bool field5 = 5;
+  optional bool field6 = 6 [default=true];
+  optional bool field7 = 7 [default=false];
+  optional float field8 = 8;
+  optional bool field9 = 9;
+  optional float field10 = 10;
+  optional int64 field11 = 11;
+}

benchmarks/google_speed.proto

@@ -0,0 +1,136 @@
+package benchmarks;
+
+option java_outer_classname = "GoogleSpeed";
+option optimize_for = SPEED;
+
+message SpeedMessage1 {
+  required string field1 = 1;
+  optional string field9 = 9;
+  optional string field18 = 18;
+  optional bool field80 = 80 [default=false];
+  optional bool field81 = 81 [default=true];
+  required int32 field2 = 2;
+  required int32 field3 = 3;
+  optional int32 field280 = 280;
+  optional int32 field6 = 6 [default=0];
+  optional int64 field22 = 22;
+  optional string field4 = 4;
+  repeated fixed64 field5 = 5;
+  optional bool field59 = 59 [default=false];
+  optional string field7 = 7;
+  optional int32 field16 = 16;
+  optional int32 field130 = 130 [default=0];
+  optional bool field12 = 12 [default=true];
+  optional bool field17 = 17 [default=true];
+  optional bool field13 = 13 [default=true];
+  optional bool field14 = 14 [default=true];
+  optional int32 field104 = 104 [default=0];
+  optional int32 field100 = 100 [default=0];
+  optional int32 field101 = 101 [default=0];
+  optional string field102 = 102;
+  optional string field103 = 103;
+  optional int32 field29 = 29 [default=0];
+  optional bool field30 = 30 [default=false];
+  optional int32 field60 = 60 [default=-1];
+  optional int32 field271 = 271 [default=-1];
+  optional int32 field272 = 272 [default=-1];
+  optional int32 field150 = 150;
+  optional int32 field23 = 23 [default=0];
+  optional bool field24 = 24 [default=false];
+  optional int32 field25 = 25 [default=0];
+  optional SpeedMessage1SubMessage field15 = 15;
+  optional bool field78 = 78;
+  optional int32 field67 = 67 [default=0];
+  optional int32 field68 = 68;
+  optional int32 field128 = 128 [default=0];
+  optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
+  optional int32 field131 = 131 [default=0];
+}
+
+message SpeedMessage1SubMessage {
+  optional int32 field1 = 1 [default=0];
+  optional int32 field2 = 2 [default=0];
+  optional int32 field3 = 3 [default=0];
+  optional string field15 = 15;
+  optional bool field12 = 12 [default=true];
+  optional int64 field13 = 13;
+  optional int64 field14 = 14;
+  optional int32 field16 = 16;
+  optional int32 field19 = 19 [default=2];
+  optional bool field20  = 20 [default=true];
+  optional bool field28 = 28 [default=true];
+  optional fixed64 field21 = 21;
+  optional int32 field22 = 22;
+  optional bool field23 = 23 [ default=false ];
+  optional bool field206 = 206 [default=false];
+  optional fixed32 field203 = 203;
+  optional int32 field204 = 204;
+  optional string field205 = 205;
+  optional uint64 field207 = 207;
+  optional uint64 field300 = 300;
+}
+
+message SpeedMessage2 {
+  optional string field1 = 1;
+  optional int64 field3 = 3;
+  optional int64 field4 = 4;
+  optional int64 field30 = 30;
+  optional bool field75  = 75 [default=false];
+  optional string field6 = 6;
+  optional bytes field2 = 2;
+  optional int32 field21 = 21 [default=0];
+  optional int32 field71 = 71;
+  optional float field25 = 25;
+  optional int32 field109 = 109 [default=0];
+  optional int32 field210 = 210 [default=0];
+  optional int32 field211 = 211 [default=0];
+  optional int32 field212 = 212 [default=0];
+  optional int32 field213 = 213 [default=0];
+  optional int32 field216 = 216 [default=0];
+  optional int32 field217 = 217 [default=0];
+  optional int32 field218 = 218 [default=0];
+  optional int32 field220 = 220 [default=0];
+  optional int32 field221 = 221 [default=0];
+  optional float field222 = 222 [default=0.0];
+  optional int32 field63 = 63;
+
+  repeated group Group1 = 10 {
+    required float field11 = 11;
+    optional float field26 = 26;
+    optional string field12 = 12;
+    optional string field13 = 13;
+    repeated string field14 = 14;
+    required uint64 field15 = 15;
+    optional int32 field5 = 5;
+    optional string field27 = 27;
+    optional int32 field28 = 28;
+    optional string field29 = 29;
+    optional string field16 = 16;
+    repeated string field22 = 22;
+    repeated int32 field73 = 73;
+    optional int32 field20 = 20 [default=0];
+    optional string field24 = 24;
+    optional SpeedMessage2GroupedMessage field31 = 31;
+  }
+  repeated string field128 = 128;
+  optional int64 field131 = 131;
+  repeated string field127 = 127;
+  optional int32 field129 = 129;
+  repeated int64 field130 = 130;
+  optional bool field205 = 205 [default=false];
+  optional bool field206 = 206 [default=false];
+}
+
+message SpeedMessage2GroupedMessage {
+  optional float field1 = 1;
+  optional float field2 = 2;
+  optional float field3 = 3 [default=0.0];
+  optional bool field4 = 4;
+  optional bool field5 = 5;
+  optional bool field6 = 6 [default=true];
+  optional bool field7 = 7 [default=false];
+  optional float field8 = 8;
+  optional bool field9 = 9;
+  optional float field10 = 10;
+  optional int64 field11 = 11;
+}

benchmarks/readme.txt

@@ -0,0 +1,50 @@
+Contents
+--------
+
+This folder contains three kinds of file:
+
+- Code, such as ProtoBench.java, to build the benchmarking framework.
+- Protocol buffer definitions (.proto files)
+- Sample data files
+
+If we end up with a lot of different benchmarks it may be worth
+separating these out info different directories, but while there are
+so few they might as well all be together.
+
+Running a benchmark (Java)
+--------------------------
+
+1) Build protoc and the Java protocol buffer library. The examples
+   below assume a jar file (protobuf.jar) has been built and copied
+   into this directory.
+
+2) Build ProtoBench:
+   $ javac -d tmp -cp protobuf.jar ProtoBench.java
+   
+3) Generate code for the relevant benchmark protocol buffer, e.g.
+   $ protoc --java_out=tmp google_size.proto google_speed.proto
+   
+4) Build the generated code, e.g.
+   $ cd tmp
+   $ javac -d . -cp ../protobuf.jar benchmarks/*.java
+           
+5) Run the test. Arguments are given in pairs - the first argument
+   is the descriptor type; the second is the filename. For example:
+   $ java -cp .;../protobuf.jar com.google.protocolbuffers.ProtoBench
+          benchmarks.GoogleSize$SizeMessage1 ../google_message1.dat
+          benchmarks.GoogleSpeed$SpeedMessage1 ../google_message1.dat
+          benchmarks.GoogleSize$SizeMessage2 ../google_message2.dat
+          benchmarks.GoogleSpeed$SpeedMessage2 ../google_message2.dat
+          
+6) Wait! Each test runs for around 30 seconds, and there are 6 tests
+   per class/data combination. The above command would therefore take
+   about 12 minutes to run.
+
+   
+Benchmarks available
+--------------------
+
+From Google:
+google_size.proto and google_speed.proto, messages
+google_message1.dat and google_message2.dat. The proto files are
+equivalent, but optimized differently.