diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
index e69d9fb7cc..aa65c02f8a 100755
--- a/benchmarks/Makefile.am
+++ b/benchmarks/Makefile.am
@@ -136,7 +136,7 @@ javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middlem
java-benchmark: javac_middleman
@echo "Writing shortcut script java-benchmark..."
@echo '#! /bin/sh' > java-benchmark
- @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark
+ @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
@chmod +x java-benchmark
java: protoc_middleman protoc_middleman2 java-benchmark
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 00a63704c1..f266d41e74 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -5,20 +5,31 @@ This directory contains benchmarking schemas and data sets that you
can use to test a variety of performance scenarios against your
protobuf language runtime.
-The schema for the datasets is described in `benchmarks.proto`.
+## Benchmark tools and build instructions
-The benchmark is based on some submodules. To initialize the submodues:
+First, you need to follow the instruction in the root directory's README to
+build your language's protobuf, then:
-For java:
-```
-$ ./initialize_submodule.sh java
-```
+### CPP
+We are using [google/benchmark](https://github.com/google/benchmark) as the
+benchmark tool for testing cpp. This is included as submodule under third_party
+directory. To init and build this tools, you need to do this under root dirctory:
-For java:
```
-$ ./initialize_submodule.sh cpp
+$ cd third_party
+$ git submodule update --init -r
+$ cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
```
+### JAVA
+We're using maven to build the java benchmarks, which is the same as to build
+the Java protobuf. There're no other tools need to install. We're using
+[google/caliper](https://github.com/google/caliper) as benchmark tool, which
+can be automaticly included by maven.
+
+
+## Run instructions
+
To run all the benchmark dataset:
For java:
@@ -49,6 +60,8 @@ $ make cpp-benchmark
$ ./cpp-benchmark $(specific generated dataset file name)
```
+## Benchmark datasets
+
There's some big testing data not included in the directory initially, you need to
run the following command to download the testing data:
@@ -57,10 +70,13 @@ $ ./download_data.sh
```
Each data set is in the format of benchmarks.proto:
+
1. name is the benchmark dataset's name.
2. message_name is the benchmark's message type full name (including package and message name)
3. payload is the list of raw data.
+The schema for the datasets is described in `benchmarks.proto`.
+
Benchmark likely want to run several benchmarks against each data set (parse,
serialize, possibly JSON, possibly using different APIs, etc).
diff --git a/benchmarks/initialize_submodule.sh b/benchmarks/initialize_submodule.sh
deleted file mode 100755
index f3456adff1..0000000000
--- a/benchmarks/initialize_submodule.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#! /bin/sh
-
-oldpwd=`pwd`
-cd "../third_party"
-git submodule update --init -r
-cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ..
-cd "$oldpwd"
diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml
index e19f3f5905..3afeebcacd 100755
--- a/benchmarks/java/pom.xml
+++ b/benchmarks/java/pom.xml
@@ -1,85 +1,59 @@
-
-
-
-
- 4.0.0
-
+
+
+ 4.0.0
protobuf-java-benchmark
- com.google.protocolbuffers
+ com.google.protobuf
1.0.0
Protocol Buffers [Benchmark]
-
- The benchmark tools for Protobuf Java.
-
-
+ The benchmark tools for Protobuf Java.
+
- com.google.protobuf
- protobuf-java
- 3.5.0
-
-
- com.google.caliper
- caliper
- 1.0-beta-2
-
+ com.google.protobuf
+ protobuf-java
+ 3.5.0
+
+
+ com.google.caliper
+ caliper
+ 1.0-beta-2
+
-
+
-
-
- org.apache.maven.plugins
- maven-assembly-plugin
- 2.4.1
-
-
-
- jar-with-dependencies
-
-
-
-
- com.mkyong.core.utils.App
-
-
-
-
-
-
- make-assembly
-
- package
-
- single
-
-
-
-
-
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 2.4.1
+
+
+
+ jar-with-dependencies
+
+
+
+
+ com.mkyong.core.utils.App
+
+
+
+
+
+ make-assembly
+
+ package
+
+ single
+
+
+
+
+
org.apache.maven.plugins
maven-compiler-plugin
3.5.1
@@ -114,35 +88,7 @@
-
- org.apache.maven.plugins
- maven-shade-plugin
- 2.3
-
-
- package
-
- shade
-
-
- true
- all
-
-
-
- com.google.monitoring.runtime.instrumentation.AllocationInstrumenter
- true
- true
-
-
-
-
-
-
-
-
-
-
+
\ No newline at end of file
diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
similarity index 92%
rename from benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java
rename to benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
index 72f2d6298e..abd9200ec3 100755
--- a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java
+++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
@@ -29,7 +29,7 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-package com.google.protocolbuffers;
+package com.google.protobuf;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
@@ -93,7 +93,7 @@ public class ProtoBench {
return false;
}
argsList.add("-DdataFile=" + file);
- argsList.add("com.google.protocolbuffers.ProtoBenchCaliper");
+ argsList.add("com.google.protobuf.ProtoBenchCaliper");
try {
String args[] = new String[argsList.size()];
@@ -121,17 +121,9 @@ public class ProtoBench {
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message3.GoogleMessage3")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
- for (String opt : ProtoBenchCaliper
- .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) {
- temp.add(opt);
- }
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message4.GoogleMessage4")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
- for (String opt : ProtoBenchCaliper
- .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) {
- temp.add(opt);
- }
} else {
return null;
}
@@ -142,6 +134,7 @@ public class ProtoBench {
String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
+ ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
temp.add(benchmarkNames);
+ temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
return temp;
}
diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
similarity index 86%
rename from benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
rename to benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
index 4045b56aed..546c25a121 100755
--- a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
+++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
@@ -1,5 +1,5 @@
-package com.google.protocolbuffers;
+package com.google.protobuf;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
@@ -26,7 +26,6 @@ public class ProtoBenchCaliper {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
.getDefaultInstance();
}
- @Override String[] getSpecificCaliperOption() { return null; }
},
GOOGLE_MESSAGE1_PROTO2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@@ -35,7 +34,6 @@ public class ProtoBenchCaliper {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
.getDefaultInstance();
}
- @Override String[] getSpecificCaliperOption() { return null; }
},
GOOGLE_MESSAGE2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@@ -43,7 +41,6 @@ public class ProtoBenchCaliper {
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
}
- @Override String[] getSpecificCaliperOption() { return null; }
},
GOOGLE_MESSAGE3 {
@Override
@@ -64,12 +61,6 @@ public class ProtoBenchCaliper {
Message getDefaultInstance() {
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
}
- @Override
- String[] getSpecificCaliperOption() {
- String[] opt = new String[1];
- opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
- return opt;
- }
},
GOOGLE_MESSAGE4 {
@Override
@@ -85,17 +76,10 @@ public class ProtoBenchCaliper {
Message getDefaultInstance() {
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
}
- @Override
- String[] getSpecificCaliperOption() {
- String[] opt = new String[1];
- opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
- return opt;
- }
};
abstract ExtensionRegistry getExtensionRegistry();
abstract Message getDefaultInstance();
- abstract String[] getSpecificCaliperOption();
}
@Param
@@ -112,8 +96,6 @@ public class ProtoBenchCaliper {
private List inputStringList;
private List sampleMessageList;
private int counter;
- private FileOutputStream devNull;
- private CodedOutputStream reuseDevNull;
@BeforeExperiment
void setUp() throws IOException {
@@ -135,15 +117,6 @@ public class ProtoBenchCaliper {
sampleMessageList.add(
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
}
- devNull = null;
- reuseDevNull = null;
-
- try {
- devNull = new FileOutputStream("/dev/null");
- reuseDevNull = CodedOutputStream.newInstance(devNull);
- } catch (FileNotFoundException e) {
- // ignore: this is probably Windows, where /dev/null does not exist
- }
counter = 0;
}
diff --git a/tests.sh b/tests.sh
index 2a0d203ba2..0a91e4ce2f 100755
--- a/tests.sh
+++ b/tests.sh
@@ -44,7 +44,10 @@ build_cpp() {
# appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
if [[ $(type cmake 2>/dev/null) ]]; then
# Verify benchmarking code can build successfully.
- cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd ..
+ git submodule init
+ git submodule update
+ cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
+ cd benchmarks && make cpp-benchmark && cd ..
else
echo ""
echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."