diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..55a47ceafe --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/benchmark"] + path = third_party/benchmark + url = https://github.com/google/benchmark.git diff --git a/.travis.yml b/.travis.yml index 094235e028..14b7050c3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ sudo: required +dist: trusty # Note: travis currently does not support listing more than one language so # this cheats and claims to only be cpp. If they add multiple language # support, this should probably get updated to install steps and/or diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index f730afe576..d98eae5ead 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -16,7 +16,9 @@ benchmarks_protoc_outputs_proto2 = \ benchmark_messages_proto2.pb.cc \ benchmark_messages_proto2.pb.h -bin_PROGRAMS = generate-datasets +AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare + +bin_PROGRAMS = generate-datasets cpp-benchmark generate_datasets_LDADD = $(top_srcdir)/src/libprotobuf.la generate_datasets_SOURCES = generate_datasets.cc @@ -30,6 +32,13 @@ nodist_generate_datasets_SOURCES = \ # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually generate_datasets-generate_datasets.$(OBJEXT): benchmarks.pb.h +cpp_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp_benchmark_SOURCES = cpp_benchmark.cc +cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_party/benchmark/include +nodist_cpp_benchmark_SOURCES = \ + $(benchmarks_protoc_outputs) \ + $(benchmarks_protoc_outputs_proto2) + $(benchmarks_protoc_outputs): protoc_middleman $(benchmarks_protoc_outputs_proto2): protoc_middleman2 diff --git a/benchmarks/benchmark_messages_proto2.proto b/benchmarks/benchmark_messages_proto2.proto index 01f67a1af3..590855033c 100644 --- a/benchmarks/benchmark_messages_proto2.proto +++ b/benchmarks/benchmark_messages_proto2.proto @@ -8,6 +8,8 @@ option java_package = "com.google.protobuf.benchmarks"; // This is the default, but we specify it here explicitly. option optimize_for = SPEED; +option cc_enable_arenas = true; + message GoogleMessage1 { required string field1 = 1; optional string field9 = 9; diff --git a/benchmarks/benchmark_messages_proto3.proto b/benchmarks/benchmark_messages_proto3.proto index 32f586986b..090b554be9 100644 --- a/benchmarks/benchmark_messages_proto3.proto +++ b/benchmarks/benchmark_messages_proto3.proto @@ -8,6 +8,8 @@ option java_package = "com.google.protobuf.benchmarks"; // This is the default, but we specify it here explicitly. option optimize_for = SPEED; +option cc_enable_arenas = true; + message GoogleMessage1 { string field1 = 1; string field9 = 9; diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp_benchmark.cc new file mode 100644 index 0000000000..0e6febc2f1 --- /dev/null +++ b/benchmarks/cpp_benchmark.cc @@ -0,0 +1,242 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include "benchmark/benchmark_api.h" +#include "benchmarks.pb.h" +#include "benchmark_messages_proto2.pb.h" +#include "benchmark_messages_proto3.pb.h" + +#define PREFIX "dataset." +#define SUFFIX ".pb" + +using benchmarks::BenchmarkDataset; +using google::protobuf::Arena; +using google::protobuf::Descriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::Message; +using google::protobuf::MessageFactory; + +class Fixture : public benchmark::Fixture { + public: + Fixture(const BenchmarkDataset& dataset, const std::string& suffix) { + for (int i = 0; i < dataset.payload_size(); i++) { + payloads_.push_back(dataset.payload(i)); + } + + const Descriptor* d = + DescriptorPool::generated_pool()->FindMessageTypeByName( + dataset.message_name()); + + if (!d) { + std::cerr << "Couldn't find message named '" << dataset.message_name() + << "\n"; + } + + prototype_ = MessageFactory::generated_factory()->GetPrototype(d); + SetName((dataset.name() + suffix).c_str()); + } + + protected: + std::vector payloads_; + const Message* prototype_; +}; + +class WrappingCounter { + public: + WrappingCounter(size_t limit) : value_(0), limit_(limit) {} + + size_t Next() { + size_t ret = value_; + if (++value_ == limit_) { + value_ = 0; + } + return ret; + } + + private: + size_t value_; + size_t limit_; +}; + +template +class ParseNewFixture : public Fixture { + public: + ParseNewFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_parse_new") {} + + virtual void BenchmarkCase(benchmark::State& state) { + WrappingCounter i(payloads_.size()); + size_t total = 0; + + while (state.KeepRunning()) { + T m; + const std::string& payload = payloads_[i.Next()]; + total += payload.size(); + m.ParseFromString(payload); + } + + state.SetBytesProcessed(total); + } +}; + +template +class ParseNewArenaFixture : public Fixture { + public: + ParseNewArenaFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_parse_newarena") {} + + virtual void BenchmarkCase(benchmark::State& state) { + WrappingCounter i(payloads_.size()); + size_t total = 0; + + while (state.KeepRunning()) { + Arena arena; + Message* m = Arena::CreateMessage(&arena); + const std::string& payload = payloads_[i.Next()]; + total += payload.size(); + m->ParseFromString(payload); + } + + state.SetBytesProcessed(total); + } +}; + +template +class ParseReuseFixture : public Fixture { + public: + ParseReuseFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_parse_reuse") {} + + virtual void BenchmarkCase(benchmark::State& state) { + T m; + WrappingCounter i(payloads_.size()); + size_t total = 0; + + while (state.KeepRunning()) { + const std::string& payload = payloads_[i.Next()]; + total += payload.size(); + m.ParseFromString(payload); + } + + state.SetBytesProcessed(total); + } +}; + +template +class SerializeFixture : public Fixture { + public: + SerializeFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_serialize") { + for (size_t i = 0; i < payloads_.size(); i++) { + message_.push_back(new T); + message_.back()->ParseFromString(payloads_[i]); + } + } + + ~SerializeFixture() { + for (size_t i = 0; i < message_.size(); i++) { + delete message_[i]; + } + } + + virtual void BenchmarkCase(benchmark::State& state) { + size_t total = 0; + std::string str; + WrappingCounter i(payloads_.size()); + + while (state.KeepRunning()) { + str.clear(); + message_[i.Next()]->SerializeToString(&str); + total += str.size(); + } + + state.SetBytesProcessed(total); + } + + private: + std::vector message_; +}; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" << name << + "', please make sure you are running " + "this command from the benchmarks/ " + "directory.\n"; + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); +} + +template +void RegisterBenchmarksForType(const BenchmarkDataset& dataset) { + ::benchmark::internal::RegisterBenchmarkInternal( + new ParseNewFixture(dataset)); + ::benchmark::internal::RegisterBenchmarkInternal( + new ParseReuseFixture(dataset)); + ::benchmark::internal::RegisterBenchmarkInternal( + new ParseNewArenaFixture(dataset)); + ::benchmark::internal::RegisterBenchmarkInternal( + new SerializeFixture(dataset)); +} + +void RegisterBenchmarks(const std::string& dataset_bytes) { + BenchmarkDataset dataset; + GOOGLE_CHECK(dataset.ParseFromString(dataset_bytes)); + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + RegisterBenchmarksForType(dataset); + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + RegisterBenchmarksForType(dataset); + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + RegisterBenchmarksForType(dataset); + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } +} + +int main(int argc, char *argv[]) { + glob_t glob_result; + if (glob("dataset.*.pb", 0, NULL, &glob_result) != 0) { + fprintf(stderr, "No dataset files found.\n"); + return 1; + } + + for (size_t i = 0; i < glob_result.gl_pathc; i++) { + fprintf(stderr, "Found input dataset: %s\n", glob_result.gl_pathv[i]); + RegisterBenchmarks(ReadFile(glob_result.gl_pathv[i])); + } + + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); +} diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile index 53ac38f39c..7e65bb4146 100644 --- a/jenkins/docker/Dockerfile +++ b/jenkins/docker/Dockerfile @@ -71,6 +71,8 @@ RUN apt-get update && apt-get install -y \ python3.4-dev \ # -- For Ruby -- ruby \ + # -- For C++ benchmarks -- + cmake \ && apt-get clean ################## diff --git a/tests.sh b/tests.sh index 9e95f2b7f9..7c5dd884e8 100755 --- a/tests.sh +++ b/tests.sh @@ -38,6 +38,9 @@ build_cpp() { cd conformance && make test_cpp && cd .. # Verify benchmarking code can build successfully. + git submodule init + git submodule update + cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../.. cd benchmarks && make && ./generate-datasets && cd .. } diff --git a/third_party/benchmark b/third_party/benchmark new file mode 160000 index 0000000000..360e66c1c4 --- /dev/null +++ b/third_party/benchmark @@ -0,0 +1 @@ +Subproject commit 360e66c1c4777c99402cf8cd535aa510fee16573