diff --git a/.gitmodules b/.gitmodules
index 55a47ceafe..a000a725f8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,4 @@
[submodule "third_party/benchmark"]
path = third_party/benchmark
url = https://github.com/google/benchmark.git
+
diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
old mode 100644
new mode 100755
index e9e08a0dc1..e69d9fb7cc
--- a/benchmarks/Makefile.am
+++ b/benchmarks/Makefile.am
@@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 = \
datasets/google_message4/benchmark_message4_3.proto
make_tmp_dir:
- mkdir -p 'tmp'
+ mkdir -p 'tmp/java/src/main/java'
touch make_tmp_dir
if USE_EXTERNAL_PROTOC
@@ -39,11 +39,11 @@ else
# relative to srcdir, which may not be the same as the current directory when
# building out-of-tree.
protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
- oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
+ oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) )
touch protoc_middleman
protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
- oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) )
+ oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) )
touch protoc_middleman2
endif
@@ -124,18 +124,19 @@ cpp: protoc_middleman protoc_middleman2 cpp-benchmark
############# JAVA RULES ##############
-javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2
- jar=`ls $(top_srcdir)/java/util/target/*.jar` && \
- jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
- javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"`
+java_benchmark_testing_files =
+ java/src/main/java/com/google/protocolbuffers/ProtoBench.java \
+ java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
+
+javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
+ cp -r java tmp && cd tmp/java && mvn clean compile assembly:single
+ cd ../..
@touch javac_middleman
java-benchmark: javac_middleman
@echo "Writing shortcut script java-benchmark..."
@echo '#! /bin/sh' > java-benchmark
- @jar=`ls $(top_srcdir)/java/util/target/*.jar` && \
- jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
- echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark
+ @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark
@chmod +x java-benchmark
java: protoc_middleman protoc_middleman2 java-benchmark
diff --git a/benchmarks/ProtoBench.java b/benchmarks/ProtoBench.java
deleted file mode 100644
index cde9095d3c..0000000000
--- a/benchmarks/ProtoBench.java
+++ /dev/null
@@ -1,319 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2009 Google Inc. All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-package com.google.protocolbuffers;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.ExtensionRegistry;
-import com.google.protobuf.Message;
-import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ProtoBench {
-
- private static final long MIN_SAMPLE_TIME_MS = 1 * 1000;
- private static final long TARGET_TIME_MS = 5 * 1000;
-
- private ProtoBench() {
- // Prevent instantiation
- }
-
- public static void main(String[] args) {
- if (args.length < 1) {
- System.err.println("Usage: ./java-benchmark ");
- System.err.println("input data is in the format of \"benchmarks.proto\"");
- System.exit(1);
- }
- boolean success = true;
- for (int i = 0; i < args.length; i++) {
- success &= runTest(args[i]);
- }
- System.exit(success ? 0 : 1);
- }
-
- public static ExtensionRegistry getExtensionsRegistry(BenchmarkDataset benchmarkDataset) {
- ExtensionRegistry extensions = ExtensionRegistry.newInstance();
- if (benchmarkDataset.getMessageName().equals("benchmarks.google_message3.GoogleMessage3")) {
- benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
- benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
- } else if (benchmarkDataset.getMessageName().equals(
- "benchmarks.google_message4.GoogleMessage4")) {
- benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
- benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
- benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
- benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
- }
-
- return extensions;
- }
-
- /**
- * Return an message instance for one specific dataset, and register the extensions for that
- * message.
- */
- public static Message registerBenchmarks(BenchmarkDataset benchmarkDataset) {
- if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
- return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
- .getDefaultInstance();
- } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
- return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
- .getDefaultInstance();
- } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
- return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
- } else if (benchmarkDataset.getMessageName().
- equals("benchmarks.google_message3.GoogleMessage3")) {
- return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
- } else if (benchmarkDataset.getMessageName().
- equals("benchmarks.google_message4.GoogleMessage4")) {
- return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
- } else {
- return null;
- }
- }
-
- /**
- * Runs a single test. Error messages are displayed to stderr, and the return value indicates
- * general success/failure.
- */
- public static boolean runTest(String file) {
- final Message defaultMessage;
- BenchmarkDataset benchmarkDataset;
- ExtensionRegistry extensions;
- final byte[] inputData;
-
- try {
- inputData = readAllBytes(file);
- benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
- } catch (IOException e) {
- System.err.println("Unable to get input data");
- return false;
- }
- defaultMessage = registerBenchmarks(benchmarkDataset);
- extensions = getExtensionsRegistry(benchmarkDataset);
- if (defaultMessage == null) {
- System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
- return false;
- }
- System.out.println("Benchmarking " + benchmarkDataset.getMessageName() + " with file " + file);
-
- try {
- List inputDataList = new ArrayList();
- List inputStreamList = new ArrayList();
- List inputStringList = new ArrayList();
- List sampleMessageList = new ArrayList();
-
- for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
- byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
- inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
- inputStreamList.add(new ByteArrayInputStream(benchmarkDataset.getPayload(i).toByteArray()));
- inputStringList.add(benchmarkDataset.getPayload(i));
- sampleMessageList.add(
- defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
- }
-
- FileOutputStream devNullTemp = null;
- CodedOutputStream reuseDevNullTemp = null;
- try {
- devNullTemp = new FileOutputStream("/dev/null");
- reuseDevNullTemp = CodedOutputStream.newInstance(devNullTemp);
- } catch (FileNotFoundException e) {
- // ignore: this is probably Windows, where /dev/null does not exist
- }
- final FileOutputStream devNull = devNullTemp;
- final CodedOutputStream reuseDevNull = reuseDevNullTemp;
- benchmark(
- "Serialize to byte string",
- inputData.length,
- new Action() {
- public void execute() {
- for (int i = 0; i < sampleMessageList.size(); i++) {
- sampleMessageList.get(i).toByteString();
- }
- }
- });
- benchmark(
- "Serialize to byte array",
- inputData.length,
- new Action() {
- public void execute() {
- for (int i = 0; i < sampleMessageList.size(); i++) {
- sampleMessageList.get(i).toByteString();
- }
- }
- });
- benchmark(
- "Serialize to memory stream",
- inputData.length,
- new Action() {
- public void execute() throws IOException {
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- for (int i = 0; i < sampleMessageList.size(); i++) {
- sampleMessageList.get(i).writeTo(output);
- }
- }
- });
- if (devNull != null) {
- benchmark(
- "Serialize to /dev/null with FileOutputStream",
- inputData.length,
- new Action() {
- public void execute() throws IOException {
- for (int i = 0; i < sampleMessageList.size(); i++) {
- sampleMessageList.get(i).writeTo(devNull);
- }
- }
- });
- benchmark(
- "Serialize to /dev/null reusing FileOutputStream",
- inputData.length,
- new Action() {
- public void execute() throws IOException {
- for (int i = 0; i < sampleMessageList.size(); i++) {
- sampleMessageList.get(i).writeTo(reuseDevNull);
- reuseDevNull.flush(); // force the write to the OutputStream
- }
- }
- });
- }
- benchmark(
- "Deserialize from byte string",
- inputData.length,
- new Action() {
- public void execute() throws IOException {
- for (int i = 0; i < inputStringList.size(); i++) {
- defaultMessage
- .newBuilderForType()
- .mergeFrom(inputStringList.get(i), extensions)
- .build();
- }
- }
- });
- benchmark(
- "Deserialize from byte array",
- inputData.length,
- new Action() {
- public void execute() throws IOException {
- for (int i = 0; i < inputDataList.size(); i++) {
- defaultMessage
- .newBuilderForType()
- .mergeFrom(CodedInputStream.newInstance(inputDataList.get(i)), extensions)
- .build();
- }
- }
- });
- benchmark(
- "Deserialize from memory stream",
- inputData.length,
- new Action() {
- public void execute() throws IOException {
- for (int i = 0; i < inputStreamList.size(); i++) {
- defaultMessage
- .newBuilderForType()
- .mergeFrom(CodedInputStream.newInstance(inputStreamList.get(i)), extensions)
- .build();
- inputStreamList.get(i).reset();
- }
- }
- });
- System.out.println();
- return true;
- } catch (Exception e) {
- System.err.println("Error: " + e.getMessage());
- System.err.println("Detailed exception information:");
- e.printStackTrace(System.err);
- return false;
- }
- }
-
- private static void benchmark(String name, long dataSize, Action action) throws IOException {
- // Make sure it's JITted "reasonably" hard before running the first progress test
- for (int i=0; i < 100; i++) {
- action.execute();
- }
-
- // Run it progressively more times until we've got a reasonable sample
- int iterations = 1;
- long elapsed = timeAction(action, iterations);
- while (elapsed < MIN_SAMPLE_TIME_MS) {
- iterations *= 2;
- elapsed = timeAction(action, iterations);
- }
-
- // Upscale the sample to the target time. Do this in floating point arithmetic
- // to avoid overflow issues.
- iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations);
- elapsed = timeAction(action, iterations);
- System.out.println(name + ": " + iterations + " iterations in "
- + (elapsed/1000f) + "s; "
- + (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f)
- + "MB/s");
- }
-
- private static long timeAction(Action action, int iterations) throws IOException {
- System.gc();
- long start = System.currentTimeMillis();
- for (int i = 0; i < iterations; i++) {
- action.execute();
- }
- long end = System.currentTimeMillis();
- return end - start;
- }
-
- private static byte[] readAllBytes(String filename) throws IOException {
- RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
- byte[] content = new byte[(int) file.length()];
- file.readFully(content);
- return content;
- }
-
- /**
- * Interface used to capture a single action to benchmark.
- */
- interface Action {
- void execute() throws IOException;
- }
-}
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 0273e38945..00a63704c1 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -7,6 +7,18 @@ protobuf language runtime.
The schema for the datasets is described in `benchmarks.proto`.
+The benchmark is based on some submodules. To initialize the submodues:
+
+For java:
+```
+$ ./initialize_submodule.sh java
+```
+
+For java:
+```
+$ ./initialize_submodule.sh cpp
+```
+
To run all the benchmark dataset:
For java:
@@ -26,17 +38,24 @@ To run a specific dataset:
For java:
```
-$ make java
+$ make java-benchmark
$ ./java-benchmark $(specific generated dataset file name)
```
For cpp:
```
-$ make cpp
+$ make cpp-benchmark
$ ./cpp-benchmark $(specific generated dataset file name)
```
+There's some big testing data not included in the directory initially, you need to
+run the following command to download the testing data:
+
+```
+$ ./download_data.sh
+```
+
Each data set is in the format of benchmarks.proto:
1. name is the benchmark dataset's name.
2. message_name is the benchmark's message type full name (including package and message name)
diff --git a/benchmarks/initialize_submodule.sh b/benchmarks/initialize_submodule.sh
new file mode 100755
index 0000000000..f3456adff1
--- /dev/null
+++ b/benchmarks/initialize_submodule.sh
@@ -0,0 +1,7 @@
+#! /bin/sh
+
+oldpwd=`pwd`
+cd "../third_party"
+git submodule update --init -r
+cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ..
+cd "$oldpwd"
diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml
new file mode 100755
index 0000000000..e19f3f5905
--- /dev/null
+++ b/benchmarks/java/pom.xml
@@ -0,0 +1,148 @@
+
+
+
+
+
+ 4.0.0
+
+ protobuf-java-benchmark
+ com.google.protocolbuffers
+ 1.0.0
+ Protocol Buffers [Benchmark]
+
+ The benchmark tools for Protobuf Java.
+
+
+
+
+ com.google.protobuf
+ protobuf-java
+ 3.5.0
+
+
+ com.google.caliper
+ caliper
+ 1.0-beta-2
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 2.4.1
+
+
+
+ jar-with-dependencies
+
+
+
+
+ com.mkyong.core.utils.App
+
+
+
+
+
+
+ make-assembly
+
+ package
+
+ single
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.5.1
+
+ 1.8
+ 1.8
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 2.5
+
+
+
+ true
+ com.google.protocolbuffers.ProtoBench
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 2.4
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 2.3
+
+
+ package
+
+ shade
+
+
+ true
+ all
+
+
+
+ com.google.monitoring.runtime.instrumentation.AllocationInstrumenter
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java
new file mode 100755
index 0000000000..72f2d6298e
--- /dev/null
+++ b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java
@@ -0,0 +1,155 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2009 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+package com.google.protocolbuffers;
+
+import com.google.caliper.BeforeExperiment;
+import com.google.caliper.Benchmark;
+import com.google.caliper.Param;
+import com.google.caliper.runner.CaliperMain;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.Message;
+import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class ProtoBench {
+
+ private ProtoBench() {
+ // Prevent instantiation
+ }
+
+ public static void main(String[] args) {
+ if (args.length < 1) {
+ System.err.println("Usage: ./java-benchmark ");
+ System.err.println("input data is in the format of \"benchmarks.proto\"");
+ System.exit(1);
+ }
+ boolean success = true;
+ for (int i = 0; i < args.length; i++) {
+ success &= runTest(args[i]);
+ }
+ System.exit(success ? 0 : 1);
+ }
+
+ /**
+ * Runs a single test with specific test data. Error messages are displayed to stderr,
+ * and the return value indicates general success/failure.
+ */
+ public static boolean runTest(String file) {
+ byte[] inputData;
+ BenchmarkDataset benchmarkDataset;
+ try {
+ inputData = readAllBytes(file);
+ benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+ } catch (IOException e) {
+ System.err.println("Unable to get input data");
+ return false;
+ }
+
+ List argsList = getCaliperOption(benchmarkDataset);
+ if (argsList == null) {
+ System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
+ return false;
+ }
+ argsList.add("-DdataFile=" + file);
+ argsList.add("com.google.protocolbuffers.ProtoBenchCaliper");
+
+ try {
+ String args[] = new String[argsList.size()];
+ argsList.toArray(args);
+ CaliperMain.exitlessMain(args,
+ new PrintWriter(System.out, true), new PrintWriter(System.err, true));
+ return true;
+ } catch (Exception e) {
+ System.err.println("Error: " + e.getMessage());
+ System.err.println("Detailed exception information:");
+ e.printStackTrace(System.err);
+ return false;
+ }
+ }
+
+
+ private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) {
+ List temp = new ArrayList();
+ if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
+ temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3");
+ } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
+ temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2");
+ } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
+ temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2");
+ } else if (benchmarkDataset.getMessageName().
+ equals("benchmarks.google_message3.GoogleMessage3")) {
+ temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
+ for (String opt : ProtoBenchCaliper
+ .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) {
+ temp.add(opt);
+ }
+ } else if (benchmarkDataset.getMessageName().
+ equals("benchmarks.google_message4.GoogleMessage4")) {
+ temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
+ for (String opt : ProtoBenchCaliper
+ .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) {
+ temp.add(opt);
+ }
+ } else {
+ return null;
+ }
+
+ temp.add("-i");
+ temp.add("runtime");
+ temp.add("-b");
+ String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
+ + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
+ temp.add(benchmarkNames);
+
+ return temp;
+ }
+
+ public static byte[] readAllBytes(String filename) throws IOException {
+ RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
+ byte[] content = new byte[(int) file.length()];
+ file.readFully(content);
+ return content;
+ }
+}
diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
new file mode 100755
index 0000000000..4045b56aed
--- /dev/null
+++ b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
@@ -0,0 +1,210 @@
+
+package com.google.protocolbuffers;
+
+import com.google.caliper.BeforeExperiment;
+import com.google.caliper.Benchmark;
+import com.google.caliper.Param;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.Message;
+import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ProtoBenchCaliper {
+ public enum BenchmarkMessageType {
+ GOOGLE_MESSAGE1_PROTO3 {
+ @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+ @Override
+ Message getDefaultInstance() {
+ return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
+ .getDefaultInstance();
+ }
+ @Override String[] getSpecificCaliperOption() { return null; }
+ },
+ GOOGLE_MESSAGE1_PROTO2 {
+ @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+ @Override
+ Message getDefaultInstance() {
+ return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
+ .getDefaultInstance();
+ }
+ @Override String[] getSpecificCaliperOption() { return null; }
+ },
+ GOOGLE_MESSAGE2 {
+ @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+ @Override
+ Message getDefaultInstance() {
+ return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
+ }
+ @Override String[] getSpecificCaliperOption() { return null; }
+ },
+ GOOGLE_MESSAGE3 {
+ @Override
+ ExtensionRegistry getExtensionRegistry() {
+ ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+ benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
+ benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
+ return extensions;
+ }
+ @Override
+ Message getDefaultInstance() {
+ return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
+ }
+ @Override
+ String[] getSpecificCaliperOption() {
+ String[] opt = new String[1];
+ opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
+ return opt;
+ }
+ },
+ GOOGLE_MESSAGE4 {
+ @Override
+ ExtensionRegistry getExtensionRegistry() {
+ ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+ benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
+ benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
+ benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
+ benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
+ return extensions;
+ }
+ @Override
+ Message getDefaultInstance() {
+ return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
+ }
+ @Override
+ String[] getSpecificCaliperOption() {
+ String[] opt = new String[1];
+ opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
+ return opt;
+ }
+ };
+
+ abstract ExtensionRegistry getExtensionRegistry();
+ abstract Message getDefaultInstance();
+ abstract String[] getSpecificCaliperOption();
+ }
+
+ @Param
+ private BenchmarkMessageType benchmarkMessageType;
+ @Param
+ private String dataFile;
+
+ private byte[] inputData;
+ private BenchmarkDataset benchmarkDataset;
+ private Message defaultMessage;
+ private ExtensionRegistry extensions;
+ private List inputDataList;
+ private List inputStreamList;
+ private List inputStringList;
+ private List sampleMessageList;
+ private int counter;
+ private FileOutputStream devNull;
+ private CodedOutputStream reuseDevNull;
+
+ @BeforeExperiment
+ void setUp() throws IOException {
+ inputData = ProtoBench.readAllBytes(dataFile);
+ benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+ defaultMessage = benchmarkMessageType.getDefaultInstance();
+ extensions = benchmarkMessageType.getExtensionRegistry();
+ inputDataList = new ArrayList();
+ inputStreamList = new ArrayList();
+ inputStringList = new ArrayList();
+ sampleMessageList = new ArrayList();
+
+ for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
+ byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
+ inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
+ inputStreamList.add(new ByteArrayInputStream(
+ benchmarkDataset.getPayload(i).toByteArray()));
+ inputStringList.add(benchmarkDataset.getPayload(i));
+ sampleMessageList.add(
+ defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
+ }
+ devNull = null;
+ reuseDevNull = null;
+
+ try {
+ devNull = new FileOutputStream("/dev/null");
+ reuseDevNull = CodedOutputStream.newInstance(devNull);
+ } catch (FileNotFoundException e) {
+ // ignore: this is probably Windows, where /dev/null does not exist
+ }
+
+ counter = 0;
+ }
+
+
+ @Benchmark
+ void serializeToByteString(int reps) throws IOException {
+ for (int i = 0; i < reps; i++) {
+ sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
+ counter++;
+ }
+ }
+
+ @Benchmark
+ void serializeToByteArray(int reps) throws IOException {
+ for (int i = 0; i < reps; i++) {
+ sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
+ counter++;
+ }
+ }
+
+ @Benchmark
+ void serializeToMemoryStream(int reps) throws IOException {
+ for (int i = 0; i < reps; i++) {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
+ counter++;
+ }
+ }
+
+ @Benchmark
+ void deserializeFromByteString(int reps) throws IOException {
+ for (int i = 0; i < reps; i++) {
+ defaultMessage
+ .newBuilderForType()
+ .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
+ .build();
+ counter++;
+ }
+ }
+
+ @Benchmark
+ void deserializeFromByteArray(int reps) throws IOException {
+ for (int i = 0; i < reps; i++) {
+ defaultMessage
+ .newBuilderForType()
+ .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
+ .build();
+ counter++;
+ }
+ }
+
+ @Benchmark
+ void deserializeFromMemoryStream(int reps) throws IOException {
+ for (int i = 0; i < reps; i++) {
+ defaultMessage
+ .newBuilderForType()
+ .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
+ .build();
+ inputStreamList.get(counter % inputStreamList.size()).reset();
+ counter++;
+ }
+ }
+}
diff --git a/tests.sh b/tests.sh
index 0a91e4ce2f..2a0d203ba2 100755
--- a/tests.sh
+++ b/tests.sh
@@ -44,10 +44,7 @@ build_cpp() {
# appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
if [[ $(type cmake 2>/dev/null) ]]; then
# Verify benchmarking code can build successfully.
- git submodule init
- git submodule update
- cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
- cd benchmarks && make cpp-benchmark && cd ..
+ cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd ..
else
echo ""
echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."