Merge pull request #1464 from google/benchmarks
Added framework for generating/consuming benchmarking data sets.pull/1482/head
Joshua Haberman
9 years ago
commit
07bcf21a9c
9 changed files with 370 additions and 10 deletions
@ -0,0 +1,66 @@ |
||||
|
||||
benchmarks_protoc_inputs = \
|
||||
benchmarks.proto \
|
||||
benchmark_messages_proto3.proto
|
||||
|
||||
benchmarks_protoc_inputs_proto2 = \
|
||||
benchmark_messages_proto2.proto
|
||||
|
||||
benchmarks_protoc_outputs = \
|
||||
benchmarks.pb.cc \
|
||||
benchmarks.pb.h \
|
||||
benchmark_messages_proto3.pb.cc \
|
||||
benchmark_messages_proto3.pb.h
|
||||
|
||||
benchmarks_protoc_outputs_proto2 = \
|
||||
benchmark_messages_proto2.pb.cc \
|
||||
benchmark_messages_proto2.pb.h
|
||||
|
||||
bin_PROGRAMS = generate-datasets
|
||||
|
||||
generate_datasets_LDADD = $(top_srcdir)/src/libprotobuf.la
|
||||
generate_datasets_SOURCES = generate_datasets.cc
|
||||
generate_datasets_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)
|
||||
nodist_generate_datasets_SOURCES = \
|
||||
$(benchmarks_protoc_outputs) \
|
||||
$(benchmarks_protoc_outputs_proto2)
|
||||
|
||||
# Explicit deps because BUILT_SOURCES are only done before a "make all/check"
|
||||
# so a direct "make test_cpp" could fail if parallel enough.
|
||||
# See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
|
||||
generate_datasets-generate_datasets.$(OBJEXT): benchmarks.pb.h |
||||
|
||||
$(benchmarks_protoc_outputs): protoc_middleman |
||||
$(benchmarks_protoc_outputs_proto2): protoc_middleman2 |
||||
|
||||
CLEANFILES = \
|
||||
$(benchmarks_protoc_outputs) \
|
||||
$(benchmarks_protoc_outputs_proto2) \
|
||||
protoc_middleman \
|
||||
protoc_middleman2 \
|
||||
dataset.*
|
||||
|
||||
if USE_EXTERNAL_PROTOC |
||||
|
||||
protoc_middleman: $(benchmarks_protoc_inputs) |
||||
$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. $(benchmarks_protoc_inputs)
|
||||
touch protoc_middleman
|
||||
|
||||
protoc_middleman2: $(benchmarks_protoc_inputs_proto2) |
||||
$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. $(benchmarks_protoc_inputs_proto2)
|
||||
touch protoc_middleman2
|
||||
|
||||
else |
||||
|
||||
# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is
|
||||
# relative to srcdir, which may not be the same as the current directory when
|
||||
# building out-of-tree.
|
||||
protoc_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) |
||||
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd $(benchmarks_protoc_inputs) )
|
||||
touch protoc_middleman
|
||||
|
||||
protoc_middleman2: $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) |
||||
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd $(benchmarks_protoc_inputs_proto2) )
|
||||
touch protoc_middleman
|
||||
|
||||
endif |
||||
@ -0,0 +1,28 @@ |
||||
|
||||
# Protocol Buffers Benchmarks |
||||
|
||||
This directory contains benchmarking schemas and data sets that you |
||||
can use to test a variety of performance scenarios against your |
||||
protobuf language runtime. |
||||
|
||||
The schema for the datasets is described in `benchmarks.proto`. |
||||
|
||||
Generate the data sets like so: |
||||
|
||||
``` |
||||
$ make |
||||
$ ./generate-datasets |
||||
Wrote dataset: dataset.google_message1_proto3.pb |
||||
Wrote dataset: dataset.google_message1_proto2.pb |
||||
Wrote dataset: dataset.google_message2.pb |
||||
$ |
||||
``` |
||||
|
||||
Each data set will be written to its own file. Benchmarks will |
||||
likely want to run several benchmarks against each data set (parse, |
||||
serialize, possibly JSON, possibly using different APIs, etc). |
||||
|
||||
We would like to add more data sets. In general we will favor data sets |
||||
that make the overall suite diverse without being too large or having |
||||
too many similar tests. Ideally everyone can run through the entire |
||||
suite without the test run getting too long. |
||||
@ -0,0 +1,76 @@ |
||||
// Benchmark messages for proto3. |
||||
|
||||
syntax = "proto3"; |
||||
|
||||
package benchmarks.proto3; |
||||
option java_package = "com.google.protobuf.benchmarks"; |
||||
|
||||
// This is the default, but we specify it here explicitly. |
||||
option optimize_for = SPEED; |
||||
|
||||
message GoogleMessage1 { |
||||
string field1 = 1; |
||||
string field9 = 9; |
||||
string field18 = 18; |
||||
bool field80 = 80; |
||||
bool field81 = 81; |
||||
int32 field2 = 2; |
||||
int32 field3 = 3; |
||||
int32 field280 = 280; |
||||
int32 field6 = 6; |
||||
int64 field22 = 22; |
||||
string field4 = 4; |
||||
repeated fixed64 field5 = 5; |
||||
bool field59 = 59; |
||||
string field7 = 7; |
||||
int32 field16 = 16; |
||||
int32 field130 = 130; |
||||
bool field12 = 12; |
||||
bool field17 = 17; |
||||
bool field13 = 13; |
||||
bool field14 = 14; |
||||
int32 field104 = 104; |
||||
int32 field100 = 100; |
||||
int32 field101 = 101; |
||||
string field102 = 102; |
||||
string field103 = 103; |
||||
int32 field29 = 29; |
||||
bool field30 = 30; |
||||
int32 field60 = 60; |
||||
int32 field271 = 271; |
||||
int32 field272 = 272; |
||||
int32 field150 = 150; |
||||
int32 field23 = 23; |
||||
bool field24 = 24; |
||||
int32 field25 = 25; |
||||
GoogleMessage1SubMessage field15 = 15; |
||||
bool field78 = 78; |
||||
int32 field67 = 67; |
||||
int32 field68 = 68; |
||||
int32 field128 = 128; |
||||
string field129 = 129; |
||||
int32 field131 = 131; |
||||
} |
||||
|
||||
message GoogleMessage1SubMessage { |
||||
int32 field1 = 1; |
||||
int32 field2 = 2; |
||||
int32 field3 = 3; |
||||
string field15 = 15; |
||||
bool field12 = 12; |
||||
int64 field13 = 13; |
||||
int64 field14 = 14; |
||||
int32 field16 = 16; |
||||
int32 field19 = 19; |
||||
bool field20 = 20; |
||||
bool field28 = 28; |
||||
fixed64 field21 = 21; |
||||
int32 field22 = 22; |
||||
bool field23 = 23; |
||||
bool field206 = 206; |
||||
fixed32 field203 = 203; |
||||
int32 field204 = 204; |
||||
string field205 = 205; |
||||
uint64 field207 = 207; |
||||
uint64 field300 = 300; |
||||
} |
||||
@ -0,0 +1,63 @@ |
||||
// Protocol Buffers - Google's data interchange format |
||||
// Copyright 2008 Google Inc. All rights reserved. |
||||
// https://developers.google.com/protocol-buffers/ |
||||
// |
||||
// Redistribution and use in source and binary forms, with or without |
||||
// modification, are permitted provided that the following conditions are |
||||
// met: |
||||
// |
||||
// * Redistributions of source code must retain the above copyright |
||||
// notice, this list of conditions and the following disclaimer. |
||||
// * Redistributions in binary form must reproduce the above |
||||
// copyright notice, this list of conditions and the following disclaimer |
||||
// in the documentation and/or other materials provided with the |
||||
// distribution. |
||||
// * Neither the name of Google Inc. nor the names of its |
||||
// contributors may be used to endorse or promote products derived from |
||||
// this software without specific prior written permission. |
||||
// |
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
|
||||
syntax = "proto3"; |
||||
package benchmarks; |
||||
option java_package = "com.google.protobuf.benchmarks"; |
||||
|
||||
message BenchmarkDataset { |
||||
// Name of the benchmark dataset. This should be unique across all datasets. |
||||
// Should only contain word characters: [a-zA-Z0-9_] |
||||
string name = 1; |
||||
|
||||
// Fully-qualified name of the protobuf message for this dataset. |
||||
// It will be one of the messages defined benchmark_messages_proto2.proto |
||||
// or benchmark_messages_proto3.proto. |
||||
// |
||||
// Implementations that do not support reflection can implement this with |
||||
// an explicit "if/else" chain that lists every known message defined |
||||
// in those files. |
||||
string message_name = 2; |
||||
|
||||
// The payload(s) for this dataset. They should be parsed or serialized |
||||
// in sequence, in a loop, ie. |
||||
// |
||||
// while (!benchmarkDone) { // Benchmark runner decides when to exit. |
||||
// for (i = 0; i < benchmark.payload.length; i++) { |
||||
// parse(benchmark.payload[i]) |
||||
// } |
||||
// } |
||||
// |
||||
// This is intended to let datasets include a variety of data to provide |
||||
// potentially more realistic results than just parsing the same message |
||||
// over and over. A single message parsed repeatedly could yield unusually |
||||
// good branch prediction performance. |
||||
repeated bytes payload = 3; |
||||
} |
||||
@ -0,0 +1,117 @@ |
||||
// Protocol Buffers - Google's data interchange format
|
||||
// Copyright 2008 Google Inc. All rights reserved.
|
||||
// https://developers.google.com/protocol-buffers/
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#include <fstream> |
||||
#include <iostream> |
||||
#include "benchmarks.pb.h" |
||||
|
||||
using benchmarks::BenchmarkDataset; |
||||
using google::protobuf::Descriptor; |
||||
using google::protobuf::DescriptorPool; |
||||
using google::protobuf::Message; |
||||
using google::protobuf::MessageFactory; |
||||
|
||||
std::set<std::string> names; |
||||
|
||||
const char *file_prefix = "dataset."; |
||||
const char *file_suffix = ".pb"; |
||||
|
||||
void WriteFileWithPayloads(const std::string& name, |
||||
const std::string& message_name, |
||||
const std::vector<std::string>& payload) { |
||||
if (!names.insert(name).second) { |
||||
std::cerr << "Duplicate test name: " << name << "\n"; |
||||
abort(); |
||||
} |
||||
|
||||
// First verify that this message name exists in our set of benchmark messages
|
||||
// and that these payloads are valid for the given message.
|
||||
const Descriptor* d = |
||||
DescriptorPool::generated_pool()->FindMessageTypeByName(message_name); |
||||
|
||||
if (!d) { |
||||
std::cerr << "For dataset " << name << ", no such message: " |
||||
<< message_name << "\n"; |
||||
abort(); |
||||
} |
||||
|
||||
Message* m = MessageFactory::generated_factory()->GetPrototype(d)->New(); |
||||
|
||||
for (size_t i = 0; i < payload.size(); i++) { |
||||
if (!m->ParseFromString(payload[i])) { |
||||
std::cerr << "For dataset " << name << ", payload[" << i << "] fails " |
||||
<< "to parse\n"; |
||||
abort(); |
||||
} |
||||
} |
||||
|
||||
BenchmarkDataset dataset; |
||||
dataset.set_name(name); |
||||
dataset.set_message_name(message_name); |
||||
for (size_t i = 0; i < payload.size(); i++) { |
||||
dataset.add_payload()->assign(payload[i]); |
||||
} |
||||
|
||||
std::ofstream writer; |
||||
std::string fname = file_prefix + name + file_suffix; |
||||
writer.open(fname.c_str()); |
||||
dataset.SerializeToOstream(&writer); |
||||
writer.close(); |
||||
|
||||
std::cerr << "Wrote dataset: " << fname << "\n"; |
||||
} |
||||
|
||||
void WriteFile(const std::string& name, const std::string& message_name, |
||||
const std::string& payload) { |
||||
std::vector<std::string> payloads; |
||||
payloads.push_back(payload); |
||||
WriteFileWithPayloads(name, message_name, payloads); |
||||
} |
||||
|
||||
std::string ReadFile(const std::string& name) { |
||||
std::ifstream file(name.c_str()); |
||||
GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" << name << |
||||
"', please make sure you are running " |
||||
"this command from the benchmarks/ " |
||||
"directory.\n"; |
||||
return std::string((std::istreambuf_iterator<char>(file)), |
||||
std::istreambuf_iterator<char>()); |
||||
} |
||||
|
||||
int main() { |
||||
WriteFile("google_message1_proto3", "benchmarks.proto3.GoogleMessage1", |
||||
ReadFile("google_message1.dat")); |
||||
WriteFile("google_message1_proto2", "benchmarks.proto2.GoogleMessage1", |
||||
ReadFile("google_message1.dat")); |
||||
|
||||
// Not in proto3 because it has a group, which is not supported.
|
||||
WriteFile("google_message2", "benchmarks.proto2.GoogleMessage2", |
||||
ReadFile("google_message2.dat")); |
||||
} |
||||
Loading…
Reference in new issue