From 7eb16a55503855d65c49b0a7e356a317c1325515 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 4 Aug 2009 10:12:41 -0700 Subject: [PATCH] Beginnings of a more extensive benchmarking suite. --- benchmark/benchmark.cc | 115 --------------------- benchmark/gen.sh | 59 +++++++++++ benchmark/getgraphs.py | 35 +++++++ benchmark/main.c | 32 ++++++ benchmark/parsetostruct_proto2_compiled.cc | 37 +++++++ benchmark/parsetostruct_proto2_table.cc | 44 ++++++++ benchmark/parsetostruct_upb_table.c | 67 ++++++++++++ 7 files changed, 274 insertions(+), 115 deletions(-) delete mode 100644 benchmark/benchmark.cc create mode 100755 benchmark/gen.sh create mode 100644 benchmark/getgraphs.py create mode 100644 benchmark/main.c create mode 100644 benchmark/parsetostruct_proto2_compiled.cc create mode 100644 benchmark/parsetostruct_proto2_table.cc create mode 100644 benchmark/parsetostruct_upb_table.c diff --git a/benchmark/benchmark.cc b/benchmark/benchmark.cc deleted file mode 100644 index c3f2f6e4b6..0000000000 --- a/benchmark/benchmark.cc +++ /dev/null @@ -1,115 +0,0 @@ - -#include -#include "google_messages.pb.h" -#include -#include "test_util.h" -#include "upb_context.h" -#include "upb_msg.h" - -int main () -{ - /* Initialize upb state, parse descriptor. */ - struct upb_context c; - upb_context_init(&c); - struct upb_string fds; - if(!upb_strreadfile("benchmark/google_messages.proto.pb", &fds)) { - fprintf(stderr, "Couldn't read google_speed.proto.bin.\n"); - return 1; - } - if(!upb_context_parsefds(&c, &fds)) { - fprintf(stderr, "Error parsing or resolving proto.\n"); - return 1; - } - upb_strfree(fds); - char class_name[] = "benchmarks.SpeedMessage2"; - struct upb_string proto_name; - proto_name.ptr = class_name; - proto_name.byte_len = sizeof(class_name)-1; - struct upb_symtab_entry *e = upb_context_lookup(&c, &proto_name); - if(!e || e->type != UPB_SYM_MESSAGE) { - fprintf(stderr, "Error finding symbol '" UPB_STRFMT "'.\n", - UPB_STRARG(proto_name)); - return 1; - } - - /* upb speed test, copying string. */ - struct upb_msg *m = e->ref.msg; - struct upb_msg_parse_state s; - void *data = upb_msgdata_new(m); - upb_msg_parse_init(&s, data, m, false, true); - size_t read; - struct upb_string str; - if(!upb_strreadfile("benchmark/google_message2.dat", &str)) { - fprintf(stderr, "Error reading google_message2.dat\n"); - return 1; - } - size_t total = 0; - clock_t before = clock(); - for(int i = 0; i < 2000; i++) { - upb_msg_parse_reset(&s, data, m, false, false); - upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read); - if(status != UPB_STATUS_OK && read != str.byte_len) { - fprintf(stderr, "Error. :( error=%d, read=%lu\n", status, read); - return 1; - } - total += str.byte_len; - } - double elapsed = ((double)clock() - before) / CLOCKS_PER_SEC; - fprintf(stderr, "upb parsed %sB, ", eng(total, 3, false)); - fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false)); - - /* upb speed test, referencing strings. */ - total = 0; - before = clock(); - for(int i = 0; i < 2000; i++) { - upb_msg_parse_reset(&s, data, m, false, true); - upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read); - if(status != UPB_STATUS_OK && read != str.byte_len) { - fprintf(stderr, "Error. :( error=%d, read=%lu\n", status, read); - return 1; - } - total += str.byte_len; - } - elapsed = ((double)clock() - before) / CLOCKS_PER_SEC; - fprintf(stderr, "upb(byref) parsed %sB, ", eng(total, 3, false)); - fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false)); - upb_msg_parse_free(&s); - upb_msgdata_free(data, m, true); - upb_context_free(&c); - - /* proto2 speed test, dynamic type. */ - std::string stlstr(str.ptr, str.byte_len); - upb_strfree(str); - - google::protobuf::DynamicMessageFactory factory; - const google::protobuf::Message *dynamic_msg_prototype = factory.GetPrototype(benchmarks::SpeedMessage2::descriptor()); - google::protobuf::Message *dynamic_msg = dynamic_msg_prototype->New(); - total = 0; - before = clock(); - for(int i = 0; i < 2000; i++) { - if(!dynamic_msg->ParseFromString(stlstr)) { - fprintf(stderr, "Error parsing with proto2.\n"); - return 1; - } - total += str.byte_len; - } - delete dynamic_msg; - elapsed = ((double)clock() - before) / CLOCKS_PER_SEC; - fprintf(stderr, "proto2(dynamic) parsed %sB, ", eng(total, 3, false)); - fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false)); - - /* proto2 speed test, compiled-in type. */ - benchmarks::SpeedMessage2 msg; - total = 0; - before = clock(); - for(int i = 0; i < 2000; i++) { - if(!msg.ParseFromString(stlstr)) { - fprintf(stderr, "Error parsing with proto2.\n"); - return 1; - } - total += str.byte_len; - } - elapsed = ((double)clock() - before) / CLOCKS_PER_SEC; - fprintf(stderr, "proto2 parsed %sB, ", eng(total, 3, false)); - fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false)); -} diff --git a/benchmark/gen.sh b/benchmark/gen.sh new file mode 100755 index 0000000000..288071eb23 --- /dev/null +++ b/benchmark/gen.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +cd `dirname $0` +CXXFLAGS="-O3 -msse3 -I../src -I../descriptor -Wall" +CFLAGS="-std=c99 $CXXFLAGS" +set -e +set -v + +gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message1.dat\" \ + -DBYREF=false \ + $CFLAGS \ + parsetostruct_upb_table.c -o b_parsetostruct_googlemessage1_upb_table_byval ../src/libupb.a + +gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message1.dat\" \ + -DBYREF=true \ + $CFLAGS \ + parsetostruct_upb_table.c -o b_parsetostruct_googlemessage1_upb_table_byref ../src/libupb.a + +gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message2.dat\" \ + -DBYREF=false \ + $CFLAGS \ + parsetostruct_upb_table.c -o b_parsetostruct_googlemessage2_upb_table_byval ../src/libupb.a + +gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message2.dat\" \ + -DBYREF=true \ + $CFLAGS \ + parsetostruct_upb_table.c -o b_parsetostruct_googlemessage2_upb_table_byref ../src/libupb.a + +g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage2" \ + -DMESSAGE_FILE=\"google_message2.dat\" \ + -DMESSAGE_HFILE=\"google_messages.pb.h\" \ + $CXXFLAGS \ + parsetostruct_proto2_table.cc -o b_parsetostruct_googlemessage2_proto2_table -lprotobuf -lpthread google_messages.pb.o + +g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage2" \ + -DMESSAGE_FILE=\"google_message2.dat\" \ + -DMESSAGE_HFILE=\"google_messages.pb.h\" \ + $CXXFLAGS \ + parsetostruct_proto2_compiled.cc -o b_parsetostruct_googlemessage2_proto2_compiled -lprotobuf -lpthread google_messages.pb.o + +g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage1" \ + -DMESSAGE_FILE=\"google_message1.dat\" \ + -DMESSAGE_HFILE=\"google_messages.pb.h\" \ + $CXXFLAGS \ + parsetostruct_proto2_table.cc -o b_parsetostruct_googlemessage1_proto2_table -lprotobuf -lpthread google_messages.pb.o + +g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage1" \ + -DMESSAGE_FILE=\"google_message1.dat\" \ + -DMESSAGE_HFILE=\"google_messages.pb.h\" \ + $CXXFLAGS \ + parsetostruct_proto2_compiled.cc -o b_parsetostruct_googlemessage1_proto2_compiled -lprotobuf -lpthread google_messages.pb.o diff --git a/benchmark/getgraphs.py b/benchmark/getgraphs.py new file mode 100644 index 0000000000..392b6efbf7 --- /dev/null +++ b/benchmark/getgraphs.py @@ -0,0 +1,35 @@ + +import sys + +benchmarks = {} +color_map = {'proto2_compiled': 'FF0000', + 'proto2_table': 'FF00FF', + 'upb_table_byref': '0000FF', + 'upb_table_byval': '00FF00'} +for line in sys.stdin: + name, val = line.split(': ') + components = name.split('_') + benchmark = '_'.join(components[1:3]) + variant = '_'.join(components[3:]) + if benchmark not in benchmarks: + benchmarks[benchmark] = [] + benchmarks[benchmark].append((variant, int(val))) + +def encode(x): + digits = (range(ord("A"), ord("Z")+1) + range(ord("a"), ord("z")+1) + + range(ord("0"), ord("9")+1) + [ord("."), ord("-")]) + return chr(digits[x / 64]) + chr(digits[x % 64]) + +for benchmark, values in benchmarks.items(): + def cmp(a, b): + return b[1] - a[1] + values.sort(cmp) + variants = [x[0] for x in values] + values = [x[1] for x in values] + scaling = 300 + encoded_values = [encode((x * 4096 / scaling) - 1) for x in values] + legend = "chdl=%s" % ("|".join(variants)) + colors = "chco=%s" % ("|".join([color_map[x] for x in variants])) + data = "chd=e:%s" % ("".join(encoded_values)) + url = "http://chart.apis.google.com/chart?cht=bhs&chs=500x200&chtt=%s+(MB/s)&chxt=x&chxr=0,0,%d&%s" % (benchmark, scaling, "&".join([legend, data, colors])) + print url diff --git a/benchmark/main.c b/benchmark/main.c new file mode 100644 index 0000000000..c0382878b3 --- /dev/null +++ b/benchmark/main.c @@ -0,0 +1,32 @@ + +#include +#include +#include + +static bool initialize(); +static void cleanup(); +static size_t run(); + +int main (int argc, char *argv[]) +{ + if(!initialize()) { + fprintf(stderr, "%s: failed to initialize\n", argv[0]); + return 1; + } + + size_t total_bytes = 0; + clock_t before = clock(); + for(int i = 0; true; i++) { + if((i & 0xFF) == 0 && (clock() - before > CLOCKS_PER_SEC)) break; + size_t bytes = run(); + if(bytes == 0) { + fprintf(stderr, "%s: failed.\n", argv[0]); + return 2; + } + total_bytes += bytes; + } + double elapsed = ((double)clock() - before) / CLOCKS_PER_SEC; + printf("%s: %d\n", argv[0], (int)(total_bytes / elapsed / (1 << 20))); + cleanup(); + return 0; +} diff --git a/benchmark/parsetostruct_proto2_compiled.cc b/benchmark/parsetostruct_proto2_compiled.cc new file mode 100644 index 0000000000..339e184198 --- /dev/null +++ b/benchmark/parsetostruct_proto2_compiled.cc @@ -0,0 +1,37 @@ + +#include "main.c" +#include MESSAGE_HFILE +#include +#include +#include +#include + +static std::string str; +MESSAGE_CIDENT msg; + +static bool initialize() +{ + /* Read the message data itself. */ + std::ifstream stream(MESSAGE_FILE); + if(!stream.is_open()) { + fprintf(stderr, "Error opening " MESSAGE_FILE ".\n"); + return false; + } + std::stringstream stringstream; + stringstream << stream.rdbuf(); + str = stringstream.str(); + return true; +} + +static void cleanup() +{ +} + +static size_t run() +{ + if(!msg.ParseFromString(str)) { + fprintf(stderr, "Error parsing with proto2.\n"); + return 0; + } + return str.size(); +} diff --git a/benchmark/parsetostruct_proto2_table.cc b/benchmark/parsetostruct_proto2_table.cc new file mode 100644 index 0000000000..815834e8ec --- /dev/null +++ b/benchmark/parsetostruct_proto2_table.cc @@ -0,0 +1,44 @@ + +#include "main.c" +#include +#include +#include +#include +#include MESSAGE_HFILE + +static std::string str; +static google::protobuf::DynamicMessageFactory factory; +static google::protobuf::Message *msg; + +static bool initialize() +{ + /* Read the message data itself. */ + std::ifstream stream(MESSAGE_FILE); + if(!stream.is_open()) { + fprintf(stderr, "Error opening " MESSAGE_FILE ".\n"); + return false; + } + std::stringstream stringstream; + stringstream << stream.rdbuf(); + str = stringstream.str(); + + /* Create the DynamicMessage. */ + const google::protobuf::Message *dynamic_msg_prototype = + factory.GetPrototype(MESSAGE_CIDENT::descriptor()); + msg = dynamic_msg_prototype->New(); + return true; +} + +static void cleanup() +{ + delete msg; +} + +static size_t run() +{ + if(!msg->ParseFromString(str)) { + fprintf(stderr, "Error parsing with proto2.\n"); + return 0; + } + return str.size(); +} diff --git a/benchmark/parsetostruct_upb_table.c b/benchmark/parsetostruct_upb_table.c new file mode 100644 index 0000000000..f93a6a7aa5 --- /dev/null +++ b/benchmark/parsetostruct_upb_table.c @@ -0,0 +1,67 @@ + +#include "main.c" + +#include "upb_context.h" +#include "upb_msg.h" + +static struct upb_context c; +static struct upb_string str; +static struct upb_msg_parse_state s; +static struct upb_msg *m; +static void *data; + +static bool initialize() +{ + /* Initialize upb state, parse descriptor. */ + upb_context_init(&c); + struct upb_string fds; + if(!upb_strreadfile(MESSAGE_DESCRIPTOR_FILE, &fds)) { + fprintf(stderr, "Couldn't read " MESSAGE_DESCRIPTOR_FILE ".\n"); + return false; + } + if(!upb_context_parsefds(&c, &fds)) { + fprintf(stderr, "Error importing " MESSAGE_DESCRIPTOR_FILE ".\n"); + return false; + } + upb_strfree(fds); + + char class_name[] = MESSAGE_NAME; + struct upb_string proto_name; + proto_name.ptr = class_name; + proto_name.byte_len = sizeof(class_name)-1; + struct upb_symtab_entry *e = upb_context_lookup(&c, &proto_name); + if(!e || e->type != UPB_SYM_MESSAGE) { + fprintf(stderr, "Error finding symbol '" UPB_STRFMT "'.\n", + UPB_STRARG(proto_name)); + return false; + } + + m = e->ref.msg; + data = upb_msgdata_new(m); + upb_msg_parse_init(&s, data, m, false, true); + + /* Read the message data itself. */ + if(!upb_strreadfile(MESSAGE_FILE, &str)) { + fprintf(stderr, "Error reading " MESSAGE_FILE "\n"); + return false; + } + return true; +} + +static void cleanup() +{ + upb_strfree(str); + upb_context_free(&c); +} + +static size_t run() +{ + size_t read; + upb_msg_parse_reset(&s, data, m, false, BYREF); + upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read); + if(status != UPB_STATUS_OK && read != str.byte_len) { + fprintf(stderr, "Error. :( error=%d, read=%lu\n", status, read); + return 0; + } + return read; +}