diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 3de0e380a2..5b25748858 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -25,76 +25,62 @@ make_tmp_dir: mkdir -p 'tmp/java/src/main/java' touch make_tmp_dir -if USE_EXTERNAL_PROTOC - -protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) - touch protoc_middleman - -protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs_proto2) - touch protoc_middleman2 - -else - # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) ) touch protoc_middleman protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) touch protoc_middleman2 -endif - -all_data = `find . -type f -name "dataset.*.pb"` +all_data = $$(find $(srcdir) -type f -name "dataset.*.pb" -not -path "./tmp/*") ############# CPP RULES ############## benchmarks_protoc_outputs = \ - benchmarks.pb.cc \ - datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc - + cpp/benchmarks.pb.cc \ + cpp/datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc + benchmarks_protoc_outputs_header = \ - benchmarks.pb.h \ - datasets/google_message1/proto3/benchmark_message1_proto3.pb.h + cpp/benchmarks.pb.h \ + cpp/datasets/google_message1/proto3/benchmark_message1_proto3.pb.h benchmarks_protoc_outputs_proto2_header = \ - datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ - datasets/google_message2/benchmark_message2.pb.h \ - datasets/google_message3/benchmark_message3.pb.h \ - datasets/google_message3/benchmark_message3_1.pb.h \ - datasets/google_message3/benchmark_message3_2.pb.h \ - datasets/google_message3/benchmark_message3_3.pb.h \ - datasets/google_message3/benchmark_message3_4.pb.h \ - datasets/google_message3/benchmark_message3_5.pb.h \ - datasets/google_message3/benchmark_message3_6.pb.h \ - datasets/google_message3/benchmark_message3_7.pb.h \ - datasets/google_message3/benchmark_message3_8.pb.h \ - datasets/google_message4/benchmark_message4.pb.h \ - datasets/google_message4/benchmark_message4_1.pb.h \ - datasets/google_message4/benchmark_message4_2.pb.h \ - datasets/google_message4/benchmark_message4_3.pb.h + cpp/datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ + cpp/datasets/google_message2/benchmark_message2.pb.h \ + cpp/datasets/google_message3/benchmark_message3.pb.h \ + cpp/datasets/google_message3/benchmark_message3_1.pb.h \ + cpp/datasets/google_message3/benchmark_message3_2.pb.h \ + cpp/datasets/google_message3/benchmark_message3_3.pb.h \ + cpp/datasets/google_message3/benchmark_message3_4.pb.h \ + cpp/datasets/google_message3/benchmark_message3_5.pb.h \ + cpp/datasets/google_message3/benchmark_message3_6.pb.h \ + cpp/datasets/google_message3/benchmark_message3_7.pb.h \ + cpp/datasets/google_message3/benchmark_message3_8.pb.h \ + cpp/datasets/google_message4/benchmark_message4.pb.h \ + cpp/datasets/google_message4/benchmark_message4_1.pb.h \ + cpp/datasets/google_message4/benchmark_message4_2.pb.h \ + cpp/datasets/google_message4/benchmark_message4_3.pb.h benchmarks_protoc_outputs_proto2 = \ - datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ - datasets/google_message2/benchmark_message2.pb.cc \ - datasets/google_message3/benchmark_message3.pb.cc \ - datasets/google_message3/benchmark_message3_1.pb.cc \ - datasets/google_message3/benchmark_message3_2.pb.cc \ - datasets/google_message3/benchmark_message3_3.pb.cc \ - datasets/google_message3/benchmark_message3_4.pb.cc \ - datasets/google_message3/benchmark_message3_5.pb.cc \ - datasets/google_message3/benchmark_message3_6.pb.cc \ - datasets/google_message3/benchmark_message3_7.pb.cc \ - datasets/google_message3/benchmark_message3_8.pb.cc \ - datasets/google_message4/benchmark_message4.pb.cc \ - datasets/google_message4/benchmark_message4_1.pb.cc \ - datasets/google_message4/benchmark_message4_2.pb.cc \ - datasets/google_message4/benchmark_message4_3.pb.cc + cpp/datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ + cpp/datasets/google_message2/benchmark_message2.pb.cc \ + cpp/datasets/google_message3/benchmark_message3.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_1.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_2.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_3.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_4.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_5.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_6.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_7.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_8.pb.cc \ + cpp/datasets/google_message4/benchmark_message4.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_1.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_2.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_3.pb.cc $(benchmarks_protoc_outputs): protoc_middleman @@ -109,8 +95,7 @@ initialize_submodule: cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make cd $$oldpwd touch initialize_submodule - -$(top_srcdir)/src/libprotobuf.la: initialize_submodule + $(top_srcdir)/third_party/benchmark/src/libbenchmark.a: initialize_submodule AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare @@ -118,12 +103,12 @@ AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Wov bin_PROGRAMS = cpp-benchmark cpp_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a -cpp_benchmark_SOURCES = cpp_benchmark.cc -cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_party/benchmark/include +cpp_benchmark_SOURCES = cpp/cpp_benchmark.cc +cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(top_srcdir)/third_party/benchmark/include # Explicit deps because BUILT_SOURCES are only done before a "make all/check" # so a direct "make test_cpp" could fail if parallel enough. # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually -cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp/cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a nodist_cpp_benchmark_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ @@ -142,7 +127,7 @@ java_benchmark_testing_files = \ java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2 - cp -r java tmp && cd tmp/java && mvn clean compile assembly:single + cp -r $(srcdir)/java tmp && cd tmp/java && mvn clean compile assembly:single cd ../.. @touch javac_middleman @@ -174,11 +159,11 @@ python_add_init: protoc_middleman protoc_middleman2 python_cpp_pkg_flags = `pkg-config --cflags --libs python` lib_LTLIBRARIES = libbenchmark_messages.la -libbenchmark_messages_la_SOURCES = python_benchmark_messages.cc +libbenchmark_messages_la_SOURCES = python/python_benchmark_messages.cc libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la libbenchmark_messages_la_LDFLAGS = -version-info 1:0:0 -export-dynamic -libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) $(python_cpp_pkg_flags) -libbenchmark_messages_la-libbenchmark_messages_la.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) +libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp $(python_cpp_pkg_flags) +libbenchmark_messages_la-python_benchmark_messages.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) nodist_libbenchmark_messages_la_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ @@ -191,7 +176,7 @@ python-pure-python-benchmark: python_add_init @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'python\' >> python-pure-python-benchmark - @echo cp py_benchmark.py tmp >> python-pure-python-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-pure-python-benchmark @echo python tmp/py_benchmark.py false '$$@' >> python-pure-python-benchmark @chmod +x python-pure-python-benchmark @@ -201,7 +186,7 @@ python-cpp-reflection-benchmark: python_add_init @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-reflection-benchmark - @echo cp py_benchmark.py tmp >> python-cpp-reflection-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-reflection-benchmark @echo python tmp/py_benchmark.py false '$$@' >> python-cpp-reflection-benchmark @chmod +x python-cpp-reflection-benchmark @@ -211,7 +196,7 @@ python-cpp-generated-code-benchmark: python_add_init libbenchmark_messages.la @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-generated-code-benchmark - @echo cp py_benchmark.py tmp >> python-cpp-generated-code-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-generated-code-benchmark @echo python tmp/py_benchmark.py true '$$@' >> python-cpp-generated-code-benchmark @chmod +x python-cpp-generated-code-benchmark @@ -251,52 +236,230 @@ benchmarks_protoc_inputs_proto2_message4 = \ datasets/google_message4/benchmark_message4_2.proto \ datasets/google_message4/benchmark_message4_3.proto -if USE_EXTERNAL_PROTOC - -go_protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) - touch protoc_middleman - -go_protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) - touch protoc_middleman2 - -else - -# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is -# relative to srcdir, which may not be the same as the current directory when -# building out-of-tree. -go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) +go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) ) - touch protoc_middleman - -go_protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) ) - touch protoc_middleman2 + touch go_protoc_middleman -endif - -go-benchmark: go_protoc_middleman go_protoc_middleman2 +go-benchmark: go_protoc_middleman @echo "Writing shortcut script go-benchmark..." @echo '#! /bin/sh' > go-benchmark - @echo 'mkdir tmp_cc && mv *.cc tmp_cc' >> go-benchmark - @echo 'go test -bench=. -- $$@' >> go-benchmark - @echo 'mv tmp_cc/* . && rm -rf tmp_cc' >> go-benchmark + @echo 'cd $(srcdir)/go' >> go-benchmark + @echo 'all_data=""' >> go-benchmark + @echo 'for data_file in $$@; do all_data="$$all_data ../$$data_file"; done' >> go-benchmark + @echo 'go test -bench=. -- $$all_data' >> go-benchmark + @echo 'cd ..' >> go-benchmark @chmod +x go-benchmark -go: go_protoc_middleman go_protoc_middleman2 go-benchmark +go: go_protoc_middleman go-benchmark ./go-benchmark $(all_data) ############# GO RULES END ############## +############# GOGO RULES BEGIN ############ + +cpp_no_group_benchmarks_protoc_outputs_header = \ + gogo/cpp_no_group/benchmarks.pb.h \ + gogo/cpp_no_group/datasets/google_message1/proto3/benchmark_message1_proto3.pb.h + +cpp_no_group_benchmarks_protoc_outputs = \ + gogo/cpp_no_group/benchmarks.pb.cc \ + gogo/cpp_no_group/datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc + +cpp_no_group_benchmarks_protoc_outputs_proto2_header = \ + gogo/cpp_no_group/datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ + gogo/cpp_no_group/datasets/google_message2/benchmark_message2.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_1.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_2.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_3.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_4.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_5.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_6.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_7.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_8.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_1.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_2.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_3.pb.h + +cpp_no_group_benchmarks_protoc_outputs_proto2 = \ + gogo/cpp_no_group/datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ + gogo/cpp_no_group/datasets/google_message2/benchmark_message2.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_1.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_2.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_3.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_4.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_5.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_6.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_7.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_8.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_1.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_2.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_3.pb.cc + +$(cpp_no_group_benchmarks_protoc_outputs): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_header): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_proto2): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_proto2_header): cpp_no_group_protoc_middleman + +generate_cpp_no_group_benchmark_code: + cp $(srcdir)/cpp/cpp_benchmark.cc gogo/cpp_no_group/cpp_benchmark.cc + sed -i -e "s/\#include \"datasets/\#include \"gogo\/cpp_no_group\/datasets/g" gogo/cpp_no_group/cpp_benchmark.cc + sed -i -e "s/\#include \"benchmarks.pb.h/\#include \"gogo\/cpp_no_group\/benchmarks.pb.h/g" gogo/cpp_no_group/cpp_benchmark.cc + touch generate_cpp_no_group_benchmark_code + +bin_PROGRAMS += cpp-no-group-benchmark +cpp_no_group_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp_no_group_benchmark_SOURCES = gogo/cpp_no_group/cpp_benchmark.cc +cpp_no_group_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/gogo/cpp_no_group -I$(top_srcdir)/third_party/benchmark/include +# Explicit deps because BUILT_SOURCES are only done before a "make all/check" +# so a direct "make test_cpp" could fail if parallel enough. +# See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +gogo/cpp_no_group/cpp_no_group_benchmark-cpp_benchmark.$(OBJEXT): $(cpp_no_group_benchmarks_protoc_outputs) $(cpp_no_group_benchmarks_protoc_outputs_proto2) $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/third_party/benchmark/src/libbenchmark.a generate_cpp_no_group_benchmark_code +gogo/cpp_no_group/cpp_benchmark.cc: generate_cpp_no_group_benchmark_code +nodist_cpp_no_group_benchmark_SOURCES = \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2) \ + $(cpp_no_group_benchmarks_protoc_outputs) \ + $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) + +cpp_no_group: cpp_no_group_protoc_middleman generate_gogo_data cpp-no-group-benchmark + ./cpp-no-group-benchmark $(gogo_data) + +gogo_proto_middleman: protoc-gen-gogoproto + mkdir -p "tmp/gogo_proto" + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-gogoproto --gogoproto_out=$$oldpwd/tmp/gogo_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) + touch gogo_proto_middleman + +new_data = $$(for data in $(all_data); do echo "tmp$${data\#$(srcdir)}"; done | xargs) + +generate_gogo_data: protoc_middleman protoc_middleman2 gogo-data-scrubber + mkdir -p `dirname $(new_data)` + ./gogo-data-scrubber $(all_data) $(new_data) + touch generate_gogo_data + +make_tmp_dir_gogo: + mkdir -p tmp/go_no_group/benchmark_code + mkdir -p tmp/gogofast/benchmark_code + mkdir -p tmp/gogofaster/benchmark_code + mkdir -p tmp/gogoslick/benchmark_code + touch make_tmp_dir_gogo + +go_no_group_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message4) ) + touch go_no_group_protoc_middleman + +cpp_no_group_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message4) ) + touch cpp_no_group_protoc_middleman + +gogofast_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogofast_protoc_middleman + +gogofaster_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogofaster_protoc_middleman + +gogoslick_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogoslick_protoc_middleman + +gogo_data = $$(find . -type f -name "dataset.*.pb" -path "./tmp/*") + +generate-gogo-benchmark-code: + @echo '#! /bin/sh' > generate-gogo-benchmark-code + @echo 'cp $(srcdir)/go/go_benchmark_test.go tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'sed -i -e "s/\.\.\/tmp/../g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'sed -i -e "s/b\.Run(\"\(.*\)\"/b.Run(\"\1\_$$1\"/g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'if [[ $$2 == 1 ]]; then sed -i -e "s/github\.com\/golang/github.com\/gogo/g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go; fi ' >> generate-gogo-benchmark-code + @chmod +x generate-gogo-benchmark-code + +generate_all_gogo_benchmark_code: generate-gogo-benchmark-code make_tmp_dir_gogo + ./generate-gogo-benchmark-code go_no_group 0 + ./generate-gogo-benchmark-code gogofast 1 + ./generate-gogo-benchmark-code gogofaster 1 + ./generate-gogo-benchmark-code gogoslick 1 + +gogo-benchmark: + @echo "Writing shortcut script gogo-benchmark..." + @echo '#! /bin/sh' > gogo-benchmark + @echo 'cd tmp/$$1/benchmark_code' >> gogo-benchmark + @echo 'shift' >> gogo-benchmark + @echo 'all_data=""' >> gogo-benchmark + @echo 'for data_file in $$@; do all_data="$$all_data ../../../$$data_file"; done' >> gogo-benchmark + @echo 'go test -bench=. -- $$all_data' >> gogo-benchmark + @echo 'cd ../..' >> gogo-benchmark + @chmod +x gogo-benchmark + +go_no_group: go_no_group_protoc_middleman generate_gogo_data generate_all_gogo_benchmark_code gogo-benchmark + ./gogo-benchmark go_no_group $(gogo_data) + +gogofast: gogofast_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogofast $(gogo_data) + +gogofaster: gogofaster_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogofaster $(gogo_data) + +gogoslick: gogoslick_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogoslick $(gogo_data) + + +############# GOGO RULES END ############ + + +############ UTIL RULES BEGIN ############ + +bin_PROGRAMS += protoc-gen-gogoproto gogo-data-scrubber + +protoc_gen_gogoproto_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/src/libprotoc.la +protoc_gen_gogoproto_SOURCES = util/protoc-gen-gogoproto.cc +protoc_gen_gogoproto_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util + +gogo_data_scrubber_LDADD = $(top_srcdir)/src/libprotobuf.la +gogo_data_scrubber_SOURCES = util/gogo_data_scrubber.cc +gogo_data_scrubber_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util +util/gogo_data_scrubber-gogo_data_scrubber.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) +nodist_gogo_data_scrubber_SOURCES = \ + $(benchmarks_protoc_outputs) \ + $(benchmarks_protoc_outputs_proto2) \ + $(benchmarks_protoc_outputs_proto2_header) \ + $(benchmarks_protoc_outputs_header) + +############ UTIL RULES END ############ MAINTAINERCLEANFILES = \ Makefile.in @@ -318,7 +481,30 @@ CLEANFILES = \ python-cpp-generated-code-benchmark \ go-benchmark \ go_protoc_middleman \ - go_protoc_middleman2 + make_tmp_dir_gogo \ + gogo_proto_middleman \ + generate_gogo_data \ + go_no_group_protoc_middleman \ + go_no_group \ + go-no-group-benchmark \ + $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2) \ + generate_all_gogo_benchmark_code \ + generate-gogo-benchmark-code \ + cpp_no_group_protoc_middleman \ + generate_cpp_no_group_benchmark_code \ + generate_gogo_benchmark_code \ + gogofast_protoc_middleman \ + gogofast \ + gogofaster_protoc_middleman \ + gogofaster \ + gogoslick_protoc_middleman \ + gogoslick \ + gogo-benchmark \ + gogo/cpp_no_group/cpp_benchmark.* + clean-local: -rm -rf tmp/* diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp/cpp_benchmark.cc similarity index 100% rename from benchmarks/cpp_benchmark.cc rename to benchmarks/cpp/cpp_benchmark.cc diff --git a/benchmarks/go_benchmark_test.go b/benchmarks/go/go_benchmark_test.go similarity index 90% rename from benchmarks/go_benchmark_test.go rename to benchmarks/go/go_benchmark_test.go index e747465e3c..8c741b7106 100644 --- a/benchmarks/go_benchmark_test.go +++ b/benchmarks/go/go_benchmark_test.go @@ -1,12 +1,12 @@ package main import ( - benchmarkWrapper "./tmp" - googleMessage1Proto2 "./tmp/datasets/google_message1/proto2" - googleMessage1Proto3 "./tmp/datasets/google_message1/proto3" - googleMessage2 "./tmp/datasets/google_message2" - googleMessage3 "./tmp/datasets/google_message3" - googleMessage4 "./tmp/datasets/google_message4" + benchmarkWrapper "../tmp" + googleMessage1Proto2 "../tmp/datasets/google_message1/proto2" + googleMessage1Proto3 "../tmp/datasets/google_message1/proto3" + googleMessage2 "../tmp/datasets/google_message2" + googleMessage3 "../tmp/datasets/google_message3" + googleMessage4 "../tmp/datasets/google_message4" "flag" "github.com/golang/protobuf/proto" "io/ioutil" diff --git a/benchmarks/python/__init__.py b/benchmarks/python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmarks/py_benchmark.py b/benchmarks/python/py_benchmark.py similarity index 69% rename from benchmarks/py_benchmark.py rename to benchmarks/python/py_benchmark.py index ba7a3470c0..a551ba6d93 100755 --- a/benchmarks/py_benchmark.py +++ b/benchmarks/python/py_benchmark.py @@ -2,21 +2,35 @@ import sys import os import timeit import math +import argparse import fnmatch - +import json + +parser = argparse.ArgumentParser(description="Python protobuf benchmark") +parser.add_argument("data_files", metavar="dataFile", nargs="+", + help="testing data files.") +parser.add_argument("--json", action="store_const", dest="json", + const="yes", default="no", + help="Whether to output json results") +parser.add_argument("--behavior_prefix", dest="behavior_prefix", + help="The output json format's behavior's name's prefix", + default="") +# BEGIN CPP GENERATED MESSAGE +parser.add_argument("--cpp_generated", action="store_const", + dest="cpp_generated", const="yes", default="no", + help="Whether to link generated code library") +# END CPP GENERATED MESSAGE +args = parser.parse_args() # BEGIN CPP GENERATED MESSAGE # CPP generated code must be linked before importing the generated Python code # for the descriptor can be found in the pool -if len(sys.argv) < 2: - raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") -if sys.argv[1] == "true": +if args.cpp_generated != "no": sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/.libs" ) import libbenchmark_messages sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/tmp" ) -elif sys.argv[1] != "false": - raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") # END CPP GENERATED MESSAGE + import datasets.google_message1.proto2.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 import datasets.google_message1.proto3.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2 @@ -26,19 +40,24 @@ import benchmarks_pb2 as benchmarks_pb2 def run_one_test(filename): - data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() + data = open(filename).read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) benchmark_util = Benchmark(full_iteration=len(benchmark_dataset.payload), module="py_benchmark", setup_method="init") - print "Message %s of dataset file %s" % \ - (benchmark_dataset.message_name, filename) + result={} + result["filename"] = filename + result["message_name"] = benchmark_dataset.message_name + result["benchmarks"] = {} benchmark_util.set_test_method("parse_from_benchmark") - print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) + result["benchmarks"][args.behavior_prefix + "_parse_from_benchmark"] = \ + benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) benchmark_util.set_test_method("serialize_to_benchmark") - print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) - print "" + result["benchmarks"][args.behavior_prefix + "_serialize_to_benchmark"] = \ + benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) + return result + def init(filename): global benchmark_dataset, message_class, message_list, counter @@ -66,11 +85,13 @@ def init(filename): temp.ParseFromString(one_payload) message_list.append(temp) + def parse_from_benchmark(): global counter, message_class, benchmark_dataset m = message_class().ParseFromString(benchmark_dataset.payload[counter % len(benchmark_dataset.payload)]) counter = counter + 1 + def serialize_to_benchmark(): global counter, message_list, message_class s = message_list[counter % len(benchmark_dataset.payload)].SerializeToString() @@ -108,11 +129,22 @@ class Benchmark: t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), setup=self.full_setup_code(setup_method_args), number=reps); - return "Average time for %s: %.2f ns" % \ - (self.test_method, 1.0 * t / reps * (10 ** 9)) - + return 1.0 * t / reps * (10 ** 9) + if __name__ == "__main__": - for i in range(2, len(sys.argv)): - run_one_test(sys.argv[i]) - + results = [] + for file in args.data_files: + results.append(run_one_test(file)) + + if args.json != "no": + print json.dumps(results) + else: + for result in results: + print "Message %s of dataset file %s" % \ + (result["message_name"], result["filename"]) + print "Average time for parse_from_benchmark: %.2f ns" % \ + (result["benchmarks"]["parse_from_benchmark"]) + print "Average time for serialize_to_benchmark: %.2f ns" % \ + (result["benchmarks"]["serialize_to_benchmark"]) + print "" diff --git a/benchmarks/python_benchmark_messages.cc b/benchmarks/python/python_benchmark_messages.cc similarity index 84% rename from benchmarks/python_benchmark_messages.cc rename to benchmarks/python/python_benchmark_messages.cc index 55242a2ac6..ded16fe96e 100644 --- a/benchmarks/python_benchmark_messages.cc +++ b/benchmarks/python/python_benchmark_messages.cc @@ -1,8 +1,8 @@ #include #include "benchmarks.pb.h" -#include "datasets/google_message1/benchmark_message1_proto2.pb.h" -#include "datasets/google_message1/benchmark_message1_proto3.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" #include "datasets/google_message2/benchmark_message2.pb.h" #include "datasets/google_message3/benchmark_message3.pb.h" #include "datasets/google_message4/benchmark_message4.pb.h" diff --git a/benchmarks/util/gogo_data_scrubber.cc b/benchmarks/util/gogo_data_scrubber.cc new file mode 100644 index 0000000000..fb9af6e2f2 --- /dev/null +++ b/benchmarks/util/gogo_data_scrubber.cc @@ -0,0 +1,105 @@ +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" + +#include + +using google::protobuf::FieldDescriptor; +using google::protobuf::Message; +using google::protobuf::Reflection; + + +class DataGroupStripper { + public: + static void StripMessage(Message *message) { + std::vector set_fields; + const Reflection* reflection = message->GetReflection(); + reflection->ListFields(*message, &set_fields); + + for (size_t i = 0; i < set_fields.size(); i++) { + const FieldDescriptor* field = set_fields[i]; + if (field->type() == FieldDescriptor::TYPE_GROUP) { + reflection->ClearField(message, field); + } + if (field->type() == FieldDescriptor::TYPE_MESSAGE) { + if (field->is_repeated()) { + for (int j = 0; j < reflection->FieldSize(*message, field); j++) { + StripMessage(reflection->MutableRepeatedMessage(message, field, j)); + } + } else { + StripMessage(reflection->MutableMessage(message, field)); + } + } + } + + reflection->MutableUnknownFields(message)->Clear(); + } +}; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" + << name + << "', please make sure you are running this command from the benchmarks" + << " directory.\n"; + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); +} + +int main(int argc, char *argv[]) { + if (argc % 2 == 0 || argc == 1) { + std::cerr << "Usage: [input_files] [output_file_names] where " << + "input_files are one to one mapping to output_file_names." << + std::endl; + return 1; + } + + for (int i = argc / 2; i > 0; i--) { + const std::string &input_file = argv[i]; + const std::string &output_file = argv[i + argc / 2]; + + std::cerr << "Generating " << input_file + << " to " << output_file << std::endl; + benchmarks::BenchmarkDataset dataset; + Message* message; + std::string dataset_payload = ReadFile(input_file); + GOOGLE_CHECK(dataset.ParseFromString(dataset_payload)) + << "Can' t parse data file " << input_file; + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + message = new benchmarks::proto3::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + message = new benchmarks::proto2::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + message = new benchmarks::proto2::GoogleMessage2; + } else if (dataset.message_name() == + "benchmarks.google_message3.GoogleMessage3") { + message = new benchmarks::google_message3::GoogleMessage3; + } else if (dataset.message_name() == + "benchmarks.google_message4.GoogleMessage4") { + message = new benchmarks::google_message4::GoogleMessage4; + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } + + for (int i = 0; i < dataset.payload_size(); i++) { + message->ParseFromString(dataset.payload(i)); + DataGroupStripper::StripMessage(message); + dataset.set_payload(i, message->SerializeAsString()); + } + + std::ofstream ofs(output_file); + ofs << dataset.SerializeAsString(); + ofs.close(); + } + + + return 0; +} diff --git a/benchmarks/util/protoc-gen-gogoproto.cc b/benchmarks/util/protoc-gen-gogoproto.cc new file mode 100644 index 0000000000..bfa6a5e54a --- /dev/null +++ b/benchmarks/util/protoc-gen-gogoproto.cc @@ -0,0 +1,103 @@ +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/printer.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "schema_proto2_to_proto3_util.h" + +#include "google/protobuf/compiler/plugin.h" + +using google::protobuf::FileDescriptorProto; +using google::protobuf::FileDescriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::io::Printer; +using google::protobuf::util::SchemaGroupStripper; +using google::protobuf::util::SchemaAddZeroEnumValue; + +namespace google { +namespace protobuf { +namespace compiler { + +namespace { + +string StripProto(string filename) { + if (filename.substr(filename.size() - 11) == ".protodevel") { + // .protodevel + return filename.substr(0, filename.size() - 11); + } else { + // .proto + return filename.substr(0, filename.size() - 6); + } +} + +DescriptorPool new_pool_; + +} // namespace + +class GoGoProtoGenerator : public CodeGenerator { + public: + virtual bool GenerateAll(const std::vector& files, + const string& parameter, + GeneratorContext* context, + string* error) const { + for (int i = 0; i < files.size(); i++) { + for (auto file : files) { + bool can_generate = + (new_pool_.FindFileByName(file->name()) == nullptr); + for (int j = 0; j < file->dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->dependency(j)->name()) != nullptr); + } + for (int j = 0; j < file->public_dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->public_dependency(j)->name()) != nullptr); + } + for (int j = 0; j < file->weak_dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->weak_dependency(j)->name()) != nullptr); + } + if (can_generate) { + Generate(file, parameter, context, error); + break; + } + } + } + + return true; + } + + virtual bool Generate(const FileDescriptor* file, + const string& parameter, + GeneratorContext* context, + string* error) const { + FileDescriptorProto new_file; + file->CopyTo(&new_file); + SchemaGroupStripper::StripFile(file, &new_file); + + SchemaAddZeroEnumValue enum_scrubber; + enum_scrubber.ScrubFile(&new_file); + + string filename = file->name(); + string basename = StripProto(filename); + + std::vector> option_pairs; + ParseGeneratorParameter(parameter, &option_pairs); + + std::unique_ptr output( + context->Open(basename + ".proto")); + string content = new_pool_.BuildFile(new_file)->DebugString(); + Printer printer(output.get(), '$'); + printer.WriteRaw(content.c_str(), content.size()); + + return true; + } +}; + +} // namespace compiler +} // namespace protobuf +} // namespace google + +int main(int argc, char* argv[]) { + google::protobuf::compiler::GoGoProtoGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/benchmarks/util/schema_proto2_to_proto3_util.h b/benchmarks/util/schema_proto2_to_proto3_util.h new file mode 100644 index 0000000000..089012dd7c --- /dev/null +++ b/benchmarks/util/schema_proto2_to_proto3_util.h @@ -0,0 +1,137 @@ +#ifndef PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ +#define PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" + +#include +#include + +using google::protobuf::Descriptor; +using google::protobuf::DescriptorProto; +using google::protobuf::FileDescriptorProto; +using google::protobuf::FieldDescriptorProto; +using google::protobuf::Message; +using google::protobuf::EnumValueDescriptorProto; + +namespace google { +namespace protobuf { +namespace util { + +class SchemaGroupStripper { + + public: + static void StripFile(const FileDescriptor* old_file, + FileDescriptorProto *file) { + for (int i = file->mutable_message_type()->size() - 1; i >= 0; i--) { + if (IsMessageSet(old_file->message_type(i))) { + file->mutable_message_type()->DeleteSubrange(i, 1); + continue; + } + StripMessage(old_file->message_type(i), file->mutable_message_type(i)); + } + for (int i = file->mutable_extension()->size() - 1; i >= 0; i--) { + auto field = old_file->extension(i); + if (field->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(field->message_type()) || + IsMessageSet(field->containing_type())) { + file->mutable_extension()->DeleteSubrange(i, 1); + } + } + } + + private: + static bool IsMessageSet(const Descriptor *descriptor) { + if (descriptor != nullptr + && descriptor->options().message_set_wire_format()) { + return true; + } + return false; + } + + static void StripMessage(const Descriptor *old_message, + DescriptorProto *new_message) { + for (int i = new_message->mutable_field()->size() - 1; i >= 0; i--) { + if (old_message->field(i)->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(old_message->field(i)->message_type())) { + new_message->mutable_field()->DeleteSubrange(i, 1); + } + } + for (int i = new_message->mutable_extension()->size() - 1; i >= 0; i--) { + auto field_type_name = new_message->mutable_extension(i)->type_name(); + if (old_message->extension(i)->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(old_message->extension(i)->containing_type()) || + IsMessageSet(old_message->extension(i)->message_type())) { + new_message->mutable_extension()->DeleteSubrange(i, 1); + } + } + for (int i = 0; i < new_message->mutable_nested_type()->size(); i++) { + StripMessage(old_message->nested_type(i), + new_message->mutable_nested_type(i)); + } + } + +}; + +class SchemaAddZeroEnumValue { + + public: + SchemaAddZeroEnumValue() + : total_added_(0) { + } + + void ScrubFile(FileDescriptorProto *file) { + for (int i = 0; i < file->enum_type_size(); i++) { + ScrubEnum(file->mutable_enum_type(i)); + } + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + ScrubMessage(file->mutable_message_type(i)); + } + } + + private: + void ScrubEnum(EnumDescriptorProto *enum_type) { + if (enum_type->value(0).number() != 0) { + bool has_zero = false; + for (int j = 0; j < enum_type->value().size(); j++) { + if (enum_type->value(j).number() == 0) { + EnumValueDescriptorProto temp_enum_value; + temp_enum_value.CopyFrom(enum_type->value(j)); + enum_type->mutable_value(j)->CopyFrom(enum_type->value(0)); + enum_type->mutable_value(0)->CopyFrom(temp_enum_value); + has_zero = true; + break; + } + } + if (!has_zero) { + enum_type->mutable_value()->Add(); + for (int i = enum_type->mutable_value()->size() - 1; i > 0; i--) { + enum_type->mutable_value(i)->CopyFrom( + *enum_type->mutable_value(i - 1)); + } + enum_type->mutable_value(0)->set_number(0); + enum_type->mutable_value(0)->set_name("ADDED_ZERO_VALUE_" + + std::to_string(total_added_++)); + } + } + + } + + void ScrubMessage(DescriptorProto *message_type) { + for (int i = 0; i < message_type->mutable_enum_type()->size(); i++) { + ScrubEnum(message_type->mutable_enum_type(i)); + } + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + ScrubMessage(message_type->mutable_nested_type(i)); + } + } + + int total_added_; +}; + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_