From 0dee28e95a4b1b7169320e05a4378af07d58c838 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 24 Apr 2015 16:10:00 -0700 Subject: [PATCH 01/17] Renamed make variables for protoc presence and version checking. --- Makefile | 10 +++++----- templates/Makefile.template | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 8cd71600ade..4d18c4ad183 100644 --- a/Makefile +++ b/Makefile @@ -348,8 +348,8 @@ OPENSSL_ALPN_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/ope ZLIB_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/zlib.c -lz $(LDFLAGS) PERFTOOLS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/perftools.c -lprofiler $(LDFLAGS) PROTOBUF_CHECK_CMD = $(CXX) $(CXXFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS) -PROTOC_CMD = which protoc > /dev/null -PROTOC_CHECK_CMD = protoc --version | grep -q libprotoc.3 +PROTOC_CHECK_CMD = which protoc > /dev/null +PROTOC_CHECK_VERSION_CMD = protoc --version | grep -q libprotoc.3 ifeq ($(OPENSSL_REQUIRES_DL),true) OPENSSL_ALPN_CHECK_CMD += -ldl @@ -375,9 +375,9 @@ HAS_SYSTEM_ZLIB = false HAS_SYSTEM_PROTOBUF = false endif -HAS_PROTOC = $(shell $(PROTOC_CMD) 2> /dev/null && echo true || echo false) +HAS_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) ifeq ($(HAS_PROTOC),true) -HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_VERSION_CMD) 2> /dev/null && echo true || echo false) else HAS_VALID_PROTOC = false endif @@ -1212,7 +1212,7 @@ run_dep_checks: $(ZLIB_CHECK_CMD) || true $(PERFTOOLS_CHECK_CMD) || true $(PROTOBUF_CHECK_CMD) || true - $(PROTOC_CHECK_CMD) || true + $(PROTOC_CHECK_VERSION_CMD) || true $(LIBDIR)/$(CONFIG)/zlib/libz.a: $(E) "[MAKE] Building zlib" diff --git a/templates/Makefile.template b/templates/Makefile.template index 9ca55f4f246..21c29454c3e 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -362,8 +362,8 @@ OPENSSL_ALPN_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/ope ZLIB_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/zlib.c -lz $(LDFLAGS) PERFTOOLS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/perftools.c -lprofiler $(LDFLAGS) PROTOBUF_CHECK_CMD = $(CXX) $(CXXFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS) -PROTOC_CMD = which protoc > /dev/null -PROTOC_CHECK_CMD = protoc --version | grep -q libprotoc.3 +PROTOC_CHECK_CMD = which protoc > /dev/null +PROTOC_CHECK_VERSION_CMD = protoc --version | grep -q libprotoc.3 ifeq ($(OPENSSL_REQUIRES_DL),true) OPENSSL_ALPN_CHECK_CMD += -ldl @@ -389,9 +389,9 @@ HAS_SYSTEM_ZLIB = false HAS_SYSTEM_PROTOBUF = false endif -HAS_PROTOC = $(shell $(PROTOC_CMD) 2> /dev/null && echo true || echo false) +HAS_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) ifeq ($(HAS_PROTOC),true) -HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_VERSION_CMD) 2> /dev/null && echo true || echo false) else HAS_VALID_PROTOC = false endif @@ -587,7 +587,7 @@ run_dep_checks: $(ZLIB_CHECK_CMD) || true $(PERFTOOLS_CHECK_CMD) || true $(PROTOBUF_CHECK_CMD) || true - $(PROTOC_CHECK_CMD) || true + $(PROTOC_CHECK_VERSION_CMD) || true $(LIBDIR)/$(CONFIG)/zlib/libz.a: $(E) "[MAKE] Building zlib" From 611b7362c6b684045191f1d116041b7de537267e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 27 Apr 2015 15:49:31 -0700 Subject: [PATCH 02/17] Tentative stap version --- src/core/profiling/stap_probes.d | 5 +++++ src/core/profiling/timers.h | 17 +++++++++++++++++ templates/Makefile.template | 25 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 src/core/profiling/stap_probes.d diff --git a/src/core/profiling/stap_probes.d b/src/core/profiling/stap_probes.d new file mode 100644 index 00000000000..04881826d0e --- /dev/null +++ b/src/core/profiling/stap_probes.d @@ -0,0 +1,5 @@ +provider _stap { + probe timing_ns_begin(int tag); + probe timing_ns_end(int tag); +}; + diff --git a/src/core/profiling/timers.h b/src/core/profiling/timers.h index 1a6b9ffac90..8fe8a8fe996 100644 --- a/src/core/profiling/timers.h +++ b/src/core/profiling/timers.h @@ -36,6 +36,23 @@ #include +#ifdef GRPC_STAP_PROFILER +#include +/* Generated from src/core/profiling/stap_probes.d */ +#include "src/core/profiling/stap_probes.h" + +#define GRPC_STAP_TIMING_NS_BEGIN(tag) _STAP_TIMING_NS_BEGIN(tag) +#define GRPC_STAP_TIMING_NS_END(tag) _STAP_TIMING_NS_END(tag) + +#else /* !GRPC_STAP_PROFILER */ +#define GRPC_STAP_BEGIN_NS(x, s) \ + do { \ + } while (0) +#define GRPC_STAP_END_NS(x, s) \ + do { \ + } while (0) +#endif /* GRPC_STAP_PROFILER */ + #ifdef __cplusplus extern "C" { #endif diff --git a/templates/Makefile.template b/templates/Makefile.template index 21c29454c3e..f0594556029 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -197,6 +197,7 @@ DEFINES_gcov = NDEBUG prefix ?= /usr/local PROTOC = protoc +DTRACE = dtrace CONFIG ?= opt CC = $(CC_$(CONFIG)) CXX = $(CXX_$(CONFIG)) @@ -364,6 +365,8 @@ PERFTOOLS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/perfto PROTOBUF_CHECK_CMD = $(CXX) $(CXXFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS) PROTOC_CHECK_CMD = which protoc > /dev/null PROTOC_CHECK_VERSION_CMD = protoc --version | grep -q libprotoc.3 +DTRACE_CHECK_CMD = which dtrace > /dev/null +SYSTEMTAP_HEADERS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/systemtap.c $(LDFLAGS) ifeq ($(OPENSSL_REQUIRES_DL),true) OPENSSL_ALPN_CHECK_CMD += -ldl @@ -396,6 +399,19 @@ else HAS_VALID_PROTOC = false endif +# Check for Systemtap (https://sourceware.org/systemtap/), first by making sure is present +# in the system and secondly by checking for the "dtrace" binary (on Linux, this is part of the Systemtap +# distribution. It's part of the base system on BSD/Solaris machines). +HAS_SYSTEMTAP_HEADERS = $(shell $(SYSTEMTAP_HEADERS_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_DTRACE = $(shell $(DTRACE_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_SYSTEMTAP = false +ifeq ($(HAS_SYSTEMTAP_HEADERS),true) +ifeq ($(HAS_DTRACE),true) +HAS_SYSTEMTAP = true +DEFINES += GRPC_STAP_PROFILER +endif +endif + ifeq ($(wildcard third_party/openssl/ssl/ssl.h),) HAS_EMBEDDED_OPENSSL_ALPN = false else @@ -863,6 +879,15 @@ endif % endfor +ifeq ($(HAS_SYSTEMTAP),true) +$(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d + $(E) "[DTRACE] Compiling $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(DTRACE) -C -h -s $< -o $@ + +src/core/profiling/timers.h: $(GENDIR)/src/core/profiling/stap_probes.h +endif + $(OBJDIR)/$(CONFIG)/%.o : %.c $(E) "[C] Compiling $<" $(Q) mkdir -p `dirname $@` From a9eb302abaccdd3609d81ba75434cccaed49f4fd Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 27 Apr 2015 19:17:20 -0700 Subject: [PATCH 03/17] STAP annotation working. Test annotations made to client_channel.c (start_rpc) and channel_create.c (grpc_channel_create). --- Makefile | 25 +++++++++++++++++ src/core/channel/client_channel.c | 4 +++ src/core/surface/channel_create.c | 8 ++++-- test/build/systemtap.c | 42 ++++++++++++++++++++++++++++ test/core/profiling/mark_timings.stp | 32 +++++++++++++++++++++ 5 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 test/build/systemtap.c create mode 100644 test/core/profiling/mark_timings.stp diff --git a/Makefile b/Makefile index 2def6e2413b..1955e2fa48a 100644 --- a/Makefile +++ b/Makefile @@ -183,6 +183,7 @@ DEFINES_gcov = NDEBUG prefix ?= /usr/local PROTOC = protoc +DTRACE = dtrace CONFIG ?= opt CC = $(CC_$(CONFIG)) CXX = $(CXX_$(CONFIG)) @@ -350,6 +351,8 @@ PERFTOOLS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/perfto PROTOBUF_CHECK_CMD = $(CXX) $(CXXFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS) PROTOC_CHECK_CMD = which protoc > /dev/null PROTOC_CHECK_VERSION_CMD = protoc --version | grep -q libprotoc.3 +DTRACE_CHECK_CMD = which dtrace > /dev/null +SYSTEMTAP_HEADERS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/systemtap.c $(LDFLAGS) ifeq ($(OPENSSL_REQUIRES_DL),true) OPENSSL_ALPN_CHECK_CMD += -ldl @@ -382,6 +385,19 @@ else HAS_VALID_PROTOC = false endif +# Check for Systemtap (https://sourceware.org/systemtap/), first by making sure is present +# in the system and secondly by checking for the "dtrace" binary (on Linux, this is part of the Systemtap +# distribution. It's part of the base system on BSD/Solaris machines). +HAS_SYSTEMTAP_HEADERS = $(shell $(SYSTEMTAP_HEADERS_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_DTRACE = $(shell $(DTRACE_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_SYSTEMTAP = false +ifeq ($(HAS_SYSTEMTAP_HEADERS),true) +ifeq ($(HAS_DTRACE),true) +HAS_SYSTEMTAP = true +DEFINES += GRPC_STAP_PROFILER +endif +endif + ifeq ($(wildcard third_party/openssl/ssl/ssl.h),) HAS_EMBEDDED_OPENSSL_ALPN = false else @@ -2779,6 +2795,15 @@ $(GENDIR)/test/proto/test.grpc.pb.cc: test/proto/test.proto $(PROTOBUF_DEP) $(PR endif +ifeq ($(HAS_SYSTEMTAP),true) +$(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d + $(E) "[DTRACE] Compiling $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(DTRACE) -C -h -s $< -o $@ + +src/core/profiling/timers.h: $(GENDIR)/src/core/profiling/stap_probes.h +endif + $(OBJDIR)/$(CONFIG)/%.o : %.c $(E) "[C] Compiling $<" $(Q) mkdir -p `dirname $@` diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index bc481e59ca5..0c241240e28 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -39,7 +39,9 @@ #include "src/core/channel/child_channel.h" #include "src/core/channel/connected_channel.h" #include "src/core/iomgr/iomgr.h" +#include "src/core/profiling/timers.h" #include "src/core/support/string.h" + #include #include #include @@ -127,6 +129,7 @@ static void complete_activate(grpc_call_element *elem, grpc_call_op *op) { static void start_rpc(grpc_call_element *elem, grpc_call_op *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; + GRPC_STAP_TIMING_NS_BEGIN(2); gpr_mu_lock(&chand->mu); if (calld->state == CALL_CANCELLED) { gpr_mu_unlock(&chand->mu); @@ -172,6 +175,7 @@ static void start_rpc(grpc_call_element *elem, grpc_call_op *op) { grpc_transport_setup_initiate(chand->transport_setup); } } + GRPC_STAP_TIMING_NS_END(2); } static void remove_waiting_child(channel_data *chand, call_data *calld) { diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index 3104b1d00d5..73b3670ccc3 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -31,6 +31,7 @@ * */ + #include "src/core/iomgr/sockaddr.h" #include @@ -48,10 +49,12 @@ #include "src/core/iomgr/endpoint.h" #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" +#include "src/core/profiling/timers.h" +#include "src/core/support/string.h" #include "src/core/surface/channel.h" #include "src/core/surface/client.h" -#include "src/core/support/string.h" #include "src/core/transport/chttp2_transport.h" + #include #include #include @@ -195,6 +198,7 @@ grpc_channel *grpc_channel_create(const char *target, #define MAX_FILTERS 3 const grpc_channel_filter *filters[MAX_FILTERS]; int n = 0; + GRPC_STAP_TIMING_NS_BEGIN(1); filters[n++] = &grpc_client_surface_filter; if (grpc_channel_args_is_census_enabled(args)) { filters[n++] = &grpc_client_census_filter; @@ -210,6 +214,6 @@ grpc_channel *grpc_channel_create(const char *target, grpc_client_setup_create_and_attach(grpc_channel_get_channel_stack(channel), args, mdctx, initiate_setup, done_setup, s); - + GRPC_STAP_TIMING_NS_END(1); return channel; } diff --git a/test/build/systemtap.c b/test/build/systemtap.c new file mode 100644 index 00000000000..66ff38ebd6d --- /dev/null +++ b/test/build/systemtap.c @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#ifndef _SYS_SDT_H +#error "_SYS_SDT_H not defined, despite being present." +#endif + +int main() { + return 0; +} diff --git a/test/core/profiling/mark_timings.stp b/test/core/profiling/mark_timings.stp new file mode 100644 index 00000000000..a7ccc83c617 --- /dev/null +++ b/test/core/profiling/mark_timings.stp @@ -0,0 +1,32 @@ +global starts, times, times_per_tag + +probe process.mark("timing_ns_begin") { + starts[$arg1, tid()] = gettimeofday_ns(); +} + +probe process.mark("timing_ns_end") { + tag = $arg1 + t = gettimeofday_ns(); + if (s = starts[tag, tid()]) { + times[tag, tid()] <<< t-s; + delete starts[tag, tid()]; + } +} + +probe end { + printf("%15s %9s %10s %10s %10s %10s\n", "tag", "tid", "count", + "min(ns)", "avg(ns)", "max(ns)"); + foreach ([tag+, tid] in times) { + printf("%15X %9d %10d %10d %10d %10d\n", tag, tid, @count(times[tag, tid]), + @min(times[tag, tid]), @avg(times[tag, tid]), @max(times[tag, tid])); + } + + printf("Per tag average of averages\n"); + foreach ([tag+, tid] in times) { + times_per_tag[tag] <<< @avg(times[tag, tid]); + } + printf("%15s %10s %10s\n", "tag", "count", "avg(ns)"); + foreach ([tag+] in times_per_tag) { + printf("%15X %10d %10d\n", tag, @count(times_per_tag[tag]), @avg(times_per_tag[tag])); + } +} From 8954e90b72e69ef353eddb1fcfd4765a75b79111 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 09:46:33 -0700 Subject: [PATCH 04/17] Integration of Systemtap (STAP) for profiling. This commit includes a faulty Makefile to be fixed in a follow commit. --- BUILD | 6 +- Makefile | 60 +++++++++--- build.json | 3 +- src/core/channel/client_channel.c | 4 +- src/core/iomgr/pollset_posix.c | 2 +- src/core/iomgr/tcp_posix.c | 20 ++-- .../profiling/{timers.c => basic_timers.c} | 39 +++++--- src/core/profiling/stap_probes.d | 1 + src/core/profiling/stap_probes.h | 43 ++++++++ src/core/profiling/stap_timers.c | 57 +++++++++++ src/core/profiling/timers.h | 98 ++++++++++++------- src/core/surface/channel_create.c | 4 +- src/core/surface/init.c | 4 +- src/cpp/client/channel.cc | 6 +- src/cpp/common/call.cc | 8 +- src/cpp/server/server.cc | 8 +- templates/Makefile.template | 54 +++++++--- vsprojects/grpc/grpc.vcxproj | 4 +- vsprojects/grpc/grpc.vcxproj.filters | 5 +- .../grpc_unsecure/grpc_unsecure.vcxproj | 4 +- .../grpc_unsecure.vcxproj.filters | 5 +- 21 files changed, 327 insertions(+), 108 deletions(-) rename src/core/profiling/{timers.c => basic_timers.c} (77%) create mode 100644 src/core/profiling/stap_probes.h create mode 100644 src/core/profiling/stap_timers.c diff --git a/BUILD b/BUILD index e862898eaa9..d06571e7350 100644 --- a/BUILD +++ b/BUILD @@ -299,7 +299,8 @@ cc_library( "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", - "src/core/profiling/timers.c", + "src/core/profiling/basic_timers.c", + "src/core/profiling/stap_timers.c", "src/core/statistics/census_init.c", "src/core/statistics/census_log.c", "src/core/statistics/census_rpc_stats.c", @@ -508,7 +509,8 @@ cc_library( "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", - "src/core/profiling/timers.c", + "src/core/profiling/basic_timers.c", + "src/core/profiling/stap_timers.c", "src/core/statistics/census_init.c", "src/core/statistics/census_log.c", "src/core/statistics/census_rpc_stats.c", diff --git a/Makefile b/Makefile index 1955e2fa48a..f222c8551e6 100644 --- a/Makefile +++ b/Makefile @@ -87,14 +87,23 @@ CPPFLAGS_opt = -O2 LDFLAGS_opt = DEFINES_opt = NDEBUG -VALID_CONFIG_latprof = 1 -CC_latprof = $(DEFAULT_CC) -CXX_latprof = $(DEFAULT_CXX) -LD_latprof = $(DEFAULT_CC) -LDXX_latprof = $(DEFAULT_CXX) -CPPFLAGS_latprof = -O2 -DGRPC_LATENCY_PROFILER -LDFLAGS_latprof = -DEFINES_latprof = NDEBUG +VALID_CONFIG_basicprof = 1 +CC_basicprof = $(DEFAULT_CC) +CXX_basicprof = $(DEFAULT_CXX) +LD_basicprof = $(DEFAULT_CC) +LDXX_basicprof = $(DEFAULT_CXX) +CPPFLAGS_basicprof = -O2 -DGRPC_BASIC_PROFILER +LDFLAGS_basicprof = +DEFINES_basicprof = NDEBUG + +VALID_CONFIG_stapprof = 1 +CC_stapprof = $(DEFAULT_CC) +CXX_stapprof = $(DEFAULT_CXX) +LD_stapprof = $(DEFAULT_CC) +LDXX_stapprof = $(DEFAULT_CXX) +CPPFLAGS_stapprof = -O2 -DGRPC_STAP_PROFILER +LDFLAGS_stapprof = +DEFINES_stapprof = NDEBUG VALID_CONFIG_dbg = 1 CC_dbg = $(DEFAULT_CC) @@ -394,7 +403,6 @@ HAS_SYSTEMTAP = false ifeq ($(HAS_SYSTEMTAP_HEADERS),true) ifeq ($(HAS_DTRACE),true) HAS_SYSTEMTAP = true -DEFINES += GRPC_STAP_PROFILER endif endif @@ -565,6 +573,17 @@ protoc_dep_message: @echo " make run_dep_checks" @echo +systemtap_dep_error: + @echo + @echo "DEPENDENCY ERROR" + @echo + @echo "Under the '$(CONFIG)' configutation, the target you are trying " + @echo "to build requires systemtap 2.7+ (on Linux) or dtrace (on other " + @echo "platforms such as Solaris and *BSD). " + @echo + @echo "Please consult INSTALL to get more information." + @echo + stop: @false @@ -2795,13 +2814,15 @@ $(GENDIR)/test/proto/test.grpc.pb.cc: test/proto/test.proto $(PROTOBUF_DEP) $(PR endif +ifeq ($(CONFIG), stapprof) ifeq ($(HAS_SYSTEMTAP),true) $(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d $(E) "[DTRACE] Compiling $<" $(Q) mkdir -p `dirname $@` $(Q) $(DTRACE) -C -h -s $< -o $@ - -src/core/profiling/timers.h: $(GENDIR)/src/core/profiling/stap_probes.h +else +$(GENDIR)/src/core/profiling/stap_probes.h: systemtap_dep_error stop +endif endif $(OBJDIR)/$(CONFIG)/%.o : %.c @@ -2824,6 +2845,17 @@ $(OBJDIR)/$(CONFIG)/%.o : %.cc $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< +ifeq ($(CONFIG), stapprof) +ifeq ($(HAS_SYSTEMTAP),true) +$(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d + $(E) "[DTRACE] Compiling $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(DTRACE) -C -h -s $< -o $@ +else +$(GENDIR)/src/core/profiling/stap_probes.c: systemtap_dep_error stop +endif +endif + install: install_c install_cxx install-plugins install-certs verify-install @@ -3223,7 +3255,8 @@ LIBGRPC_SRC = \ src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ - src/core/profiling/timers.c \ + src/core/profiling/basic_timers.c \ + src/core/profiling/stap_timers.c \ src/core/statistics/census_init.c \ src/core/statistics/census_log.c \ src/core/statistics/census_rpc_stats.c \ @@ -3472,7 +3505,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ - src/core/profiling/timers.c \ + src/core/profiling/basic_timers.c \ + src/core/profiling/stap_timers.c \ src/core/statistics/census_init.c \ src/core/statistics/census_log.c \ src/core/statistics/census_rpc_stats.c \ diff --git a/build.json b/build.json index e072ed67c3c..5408ccc0d61 100644 --- a/build.json +++ b/build.json @@ -234,7 +234,8 @@ "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", - "src/core/profiling/timers.c", + "src/core/profiling/basic_timers.c", + "src/core/profiling/stap_timers.c", "src/core/statistics/census_init.c", "src/core/statistics/census_log.c", "src/core/statistics/census_rpc_stats.c", diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index 0c241240e28..7be89e53497 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -129,7 +129,7 @@ static void complete_activate(grpc_call_element *elem, grpc_call_op *op) { static void start_rpc(grpc_call_element *elem, grpc_call_op *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - GRPC_STAP_TIMING_NS_BEGIN(2); + GRPC_TIMER_BEGIN(PTAG_OTHER_BASE + 1, NULL); gpr_mu_lock(&chand->mu); if (calld->state == CALL_CANCELLED) { gpr_mu_unlock(&chand->mu); @@ -175,7 +175,7 @@ static void start_rpc(grpc_call_element *elem, grpc_call_op *op) { grpc_transport_setup_initiate(chand->transport_setup); } } - GRPC_STAP_TIMING_NS_END(2); + GRPC_TIMER_BEGIN(PTAG_OTHER_BASE + 1, NULL); } static void remove_waiting_child(channel_data *chand, call_data *calld) { diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 60d0dad6d87..4985dcf7399 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -411,7 +411,7 @@ static int unary_poll_pollset_maybe_work(grpc_pollset *pollset, pfd[1].events = grpc_fd_begin_poll(fd, pollset, POLLIN, POLLOUT, &fd_watcher); r = poll(pfd, GPR_ARRAY_SIZE(pfd), timeout); - GRPC_TIMER_MARK(POLL_FINISHED, r); + GRPC_TIMER_MARK(PTAG_POLL_FINISHED, r); grpc_fd_end_poll(&fd_watcher); diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index 86721e9c955..7626a511846 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -327,7 +327,7 @@ static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { gpr_slice *final_slices; size_t final_nslices; - GRPC_TIMER_MARK(HANDLE_READ_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_HANDLE_READ, 0); slice_state_init(&read_state, static_read_slices, INLINE_SLICE_BUFFER_SIZE, 0); @@ -350,11 +350,11 @@ static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { msg.msg_controllen = 0; msg.msg_flags = 0; - GRPC_TIMER_MARK(RECVMSG_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_RECVMSG, 0); do { read_bytes = recvmsg(tcp->fd, &msg, 0); } while (read_bytes < 0 && errno == EINTR); - GRPC_TIMER_MARK(RECVMSG_END, 0); + GRPC_TIMER_END(PTAG_RECVMSG, 0); if (read_bytes < allocated_bytes) { /* TODO(klempner): Consider a second read first, in hopes of getting a @@ -406,7 +406,7 @@ static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { ++iov_size; } } - GRPC_TIMER_MARK(HANDLE_READ_END, 0); + GRPC_TIMER_END(PTAG_HANDLE_READ, 0); } static void grpc_tcp_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, @@ -438,12 +438,12 @@ static grpc_endpoint_write_status grpc_tcp_flush(grpc_tcp *tcp) { msg.msg_controllen = 0; msg.msg_flags = 0; - GRPC_TIMER_MARK(SENDMSG_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_SENDMSG, 0); do { /* TODO(klempner): Cork if this is a partial write */ sent_length = sendmsg(tcp->fd, &msg, 0); } while (sent_length < 0 && errno == EINTR); - GRPC_TIMER_MARK(SENDMSG_END, 0); + GRPC_TIMER_END(PTAG_SENDMSG, 0); if (sent_length < 0) { if (errno == EAGAIN) { @@ -479,7 +479,7 @@ static void grpc_tcp_handle_write(void *arg /* grpc_tcp */, int success) { return; } - GRPC_TIMER_MARK(CB_WRITE_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_TCP_CB_WRITE, 0); write_status = grpc_tcp_flush(tcp); if (write_status == GRPC_ENDPOINT_WRITE_PENDING) { grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_closure); @@ -495,7 +495,7 @@ static void grpc_tcp_handle_write(void *arg /* grpc_tcp */, int success) { cb(tcp->write_user_data, cb_status); grpc_tcp_unref(tcp); } - GRPC_TIMER_MARK(CB_WRITE_END, 0); + GRPC_TIMER_END(PTAG_TCP_CB_WRITE, 0); } static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, @@ -518,7 +518,7 @@ static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, } } - GRPC_TIMER_MARK(WRITE_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_TCP_WRITE, 0); GPR_ASSERT(tcp->write_cb == NULL); slice_state_init(&tcp->write_state, slices, nslices, nslices); @@ -532,7 +532,7 @@ static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_closure); } - GRPC_TIMER_MARK(WRITE_END, 0); + GRPC_TIMER_END(PTAG_TCP_WRITE, 0); return status; } diff --git a/src/core/profiling/timers.c b/src/core/profiling/basic_timers.c similarity index 77% rename from src/core/profiling/timers.c rename to src/core/profiling/basic_timers.c index 7cc79bd22bc..b33afa7e8cc 100644 --- a/src/core/profiling/timers.c +++ b/src/core/profiling/basic_timers.c @@ -31,7 +31,9 @@ * */ -#ifdef GRPC_LATENCY_PROFILER +#include + +#ifdef GRPC_BASIC_PROFILER #include "src/core/profiling/timers.h" #include "src/core/profiling/timers_preciseclock.h" @@ -44,7 +46,7 @@ typedef struct grpc_timer_entry { grpc_precise_clock tm; - const char* tag; + int tag; void* id; const char* file; int line; @@ -61,7 +63,7 @@ struct grpc_timers_log { grpc_timers_log* grpc_timers_log_global = NULL; -grpc_timers_log* grpc_timers_log_create(int capacity_limit, FILE* dump) { +static grpc_timers_log* grpc_timers_log_create(int capacity_limit, FILE* dump) { grpc_timers_log* log = gpr_malloc(sizeof(*log)); /* TODO (vpai): Allow allocation below limit */ @@ -85,7 +87,7 @@ static void log_report_locked(grpc_timers_log* log) { grpc_timer_entry* entry = &(log->log[i]); fprintf(fp, "GRPC_LAT_PROF "); grpc_precise_clock_print(&entry->tm, fp); - fprintf(fp, " %s %p %s %d\n", entry->tag, entry->id, entry->file, + fprintf(fp, " %d %p %s %d\n", entry->tag, entry->id, entry->file, entry->line); } @@ -93,7 +95,7 @@ static void log_report_locked(grpc_timers_log* log) { log->num_entries = 0; } -void grpc_timers_log_destroy(grpc_timers_log* log) { +static void grpc_timers_log_destroy(grpc_timers_log* log) { gpr_mu_lock(&log->mu); log_report_locked(log); gpr_mu_unlock(&log->mu); @@ -104,8 +106,8 @@ void grpc_timers_log_destroy(grpc_timers_log* log) { gpr_free(log); } -void grpc_timers_log_add(grpc_timers_log* log, const char* tag, void* id, - const char* file, int line) { +static void grpc_timers_log_add(grpc_timers_log* log, int tag, void* id, + const char* file, int line) { grpc_timer_entry* entry; /* TODO (vpai) : Improve concurrency */ @@ -125,14 +127,25 @@ void grpc_timers_log_add(grpc_timers_log* log, const char* tag, void* id, gpr_mu_unlock(&log->mu); } -void grpc_timers_log_global_init(void) { +/* Latency profiler API implementation. */ +void grpc_timer_add_mark(int tag, void* id, const char* file, int line) { + grpc_timers_log_add(grpc_timers_log_global, tag, id, file, line); +} + +void grpc_timer_begin(int tag, void* id, const char *file, int line) {} +void grpc_timer_end(int tag, void* id, const char *file, int line) {} + +/* Basic profiler specific API functions. */ +void grpc_timers_global_init(void) { grpc_timers_log_global = grpc_timers_log_create(100000, stdout); } -void grpc_timers_log_global_destroy(void) { +void grpc_timers_global_destroy(void) { grpc_timers_log_destroy(grpc_timers_log_global); } -#else /* !GRPC_LATENCY_PROFILER */ -void grpc_timers_log_global_init(void) {} -void grpc_timers_log_global_destroy(void) {} -#endif /* GRPC_LATENCY_PROFILER */ + + +#else /* !GRPC_BASIC_PROFILER */ +void grpc_timers_global_init(void) {} +void grpc_timers_global_destroy(void) {} +#endif /* GRPC_BASIC_PROFILER */ diff --git a/src/core/profiling/stap_probes.d b/src/core/profiling/stap_probes.d index 04881826d0e..374eeedd6c0 100644 --- a/src/core/profiling/stap_probes.d +++ b/src/core/profiling/stap_probes.d @@ -1,4 +1,5 @@ provider _stap { + probe add_mark(int tag); probe timing_ns_begin(int tag); probe timing_ns_end(int tag); }; diff --git a/src/core/profiling/stap_probes.h b/src/core/profiling/stap_probes.h new file mode 100644 index 00000000000..ba8020c5a32 --- /dev/null +++ b/src/core/profiling/stap_probes.h @@ -0,0 +1,43 @@ +/* Generated by the Systemtap dtrace wrapper */ + +#define _SDT_HAS_SEMAPHORES 1 + + +#define STAP_HAS_SEMAPHORES 1 /* deprecated */ + + +#include + +/* _STAP_ADD_MARK ( int tag) */ +#if defined STAP_SDT_V1 +#define _STAP_ADD_MARK_ENABLED() __builtin_expect (add_mark_semaphore, 0) +#define _stap_add_mark_semaphore add_mark_semaphore +#else +#define _STAP_ADD_MARK_ENABLED() __builtin_expect (_stap_add_mark_semaphore, 0) +#endif +__extension__ extern unsigned short _stap_add_mark_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); +#define _STAP_ADD_MARK(arg1) \ +DTRACE_PROBE1(_stap,add_mark,arg1) + +/* _STAP_TIMING_NS_BEGIN ( int tag) */ +#if defined STAP_SDT_V1 +#define _STAP_TIMING_NS_BEGIN_ENABLED() __builtin_expect (timing_ns_begin_semaphore, 0) +#define _stap_timing_ns_begin_semaphore timing_ns_begin_semaphore +#else +#define _STAP_TIMING_NS_BEGIN_ENABLED() __builtin_expect (_stap_timing_ns_begin_semaphore, 0) +#endif +__extension__ extern unsigned short _stap_timing_ns_begin_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); +#define _STAP_TIMING_NS_BEGIN(arg1) \ +DTRACE_PROBE1(_stap,timing_ns_begin,arg1) + +/* _STAP_TIMING_NS_END ( int tag) */ +#if defined STAP_SDT_V1 +#define _STAP_TIMING_NS_END_ENABLED() __builtin_expect (timing_ns_end_semaphore, 0) +#define _stap_timing_ns_end_semaphore timing_ns_end_semaphore +#else +#define _STAP_TIMING_NS_END_ENABLED() __builtin_expect (_stap_timing_ns_end_semaphore, 0) +#endif +__extension__ extern unsigned short _stap_timing_ns_end_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); +#define _STAP_TIMING_NS_END(arg1) \ +DTRACE_PROBE1(_stap,timing_ns_end,arg1) + diff --git a/src/core/profiling/stap_timers.c b/src/core/profiling/stap_timers.c new file mode 100644 index 00000000000..52fb58a2797 --- /dev/null +++ b/src/core/profiling/stap_timers.c @@ -0,0 +1,57 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#ifdef GRPC_STAP_PROFILER + +#include "src/core/profiling/timers.h" + +#include +/* Generated from src/core/profiling/stap_probes.d */ +#include "src/core/profiling/stap_probes.h" + +/* Latency profiler API implementation. */ +void grpc_timer_add_mark(int tag, void* id, const char *file, int line) { + _STAP_ADD_MARK(tag); +} + +void grpc_timer_begin(int tag, void* id, const char *file, int line) { + _STAP_TIMING_NS_BEGIN(tag); +} + +void grpc_timer_end(int tag, void* id, const char *file, int line) { + _STAP_TIMING_NS_END(tag); +} + +#endif /* GRPC_STAP_PROFILER */ diff --git a/src/core/profiling/timers.h b/src/core/profiling/timers.h index 8fe8a8fe996..a78bbea4ec8 100644 --- a/src/core/profiling/timers.h +++ b/src/core/profiling/timers.h @@ -34,52 +34,78 @@ #ifndef GRPC_CORE_PROFILING_TIMERS_H #define GRPC_CORE_PROFILING_TIMERS_H -#include - -#ifdef GRPC_STAP_PROFILER -#include -/* Generated from src/core/profiling/stap_probes.d */ -#include "src/core/profiling/stap_probes.h" - -#define GRPC_STAP_TIMING_NS_BEGIN(tag) _STAP_TIMING_NS_BEGIN(tag) -#define GRPC_STAP_TIMING_NS_END(tag) _STAP_TIMING_NS_END(tag) - -#else /* !GRPC_STAP_PROFILER */ -#define GRPC_STAP_BEGIN_NS(x, s) \ - do { \ - } while (0) -#define GRPC_STAP_END_NS(x, s) \ - do { \ - } while (0) -#endif /* GRPC_STAP_PROFILER */ - #ifdef __cplusplus extern "C" { #endif -#ifdef GRPC_LATENCY_PROFILER +void grpc_timers_global_init(void); +void grpc_timers_global_destroy(void); -typedef struct grpc_timers_log grpc_timers_log; +void grpc_timer_add_mark(int tag, void* id, const char *file, int line); +void grpc_timer_begin(int tag, void* id, const char *file, int line); +void grpc_timer_end(int tag, void* id, const char *file, int line); -grpc_timers_log* grpc_timers_log_create(int capacity_limit, FILE* dump); -void grpc_timers_log_add(grpc_timers_log*, const char* tag, void* id, - const char* file, int line); -void grpc_timers_log_destroy(grpc_timers_log *); +enum profiling_tags { + /* Re. Protos. */ + PTAG_PROTO_SERIALIZE = 100, + PTAG_PROTO_DESERIALIZE = 101, -extern grpc_timers_log *grpc_timers_log_global; + /* Re. sockets. */ + PTAG_HANDLE_READ = 200, + PTAG_SENDMSG = 201, + PTAG_RECVMSG = 202, + PTAG_POLL_FINISHED = 203, + PTAG_TCP_CB_WRITE = 204, + PTAG_TCP_WRITE = 205, -#define GRPC_TIMER_MARK(x, s) \ - grpc_timers_log_add(grpc_timers_log_global, #x, ((void *)(gpr_intptr)(s)), \ - __FILE__, __LINE__) + /* C++ */ + PTAG_CPP_CALL_CREATED = 300, + PTAG_CPP_PERFORM_OPS = 301, -#else /* !GRPC_LATENCY_PROFILER */ -#define GRPC_TIMER_MARK(x, s) \ - do { \ - } while (0) -#endif /* GRPC_LATENCY_PROFILER */ + /* > 1024 Unassigned reserved. For any miscellaneous use. + * Use addition to generate tags from this base or take advantage of the 10 + * zero'd bits for OR-ing. */ + PTAG_OTHER_BASE = 1024 +}; + +#if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER)) +/* No profiling. No-op all the things. */ +#define GRPC_TIMER_MARK(tag, id) \ + do {} while(0) + +#define GRPC_TIMER_BEGIN(tag, id) \ + do {} while(0) + +#define GRPC_TIMER_END(tag, id) \ + do {} while(0) + +#else /* at least one profiler requested... */ +/* ... hopefully only one. */ +#if defined(GRPC_STAP_PROFILER) && defined(GRPC_BASIC_PROFILER) +#error "GRPC_STAP_PROFILER and GRPC_BASIC_PROFILER are mutually exclusive." +#endif + +/* Generic profiling interface. */ +#define GRPC_TIMER_MARK(tag, id) \ + grpc_timer_add_mark(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__) + +#define GRPC_TIMER_BEGIN(tag, id) \ + grpc_timer_begin(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__) + +#define GRPC_TIMER_END(tag, id) \ + grpc_timer_end(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__) + +#ifdef GRPC_STAP_PROFILER +/* Empty placeholder for now. */ +#endif /* GRPC_STAP_PROFILER */ + +#ifdef GRPC_BASIC_PROFILER +typedef struct grpc_timers_log grpc_timers_log; + +extern grpc_timers_log *grpc_timers_log_global; +#endif /* GRPC_BASIC_PROFILER */ -void grpc_timers_log_global_init(void); -void grpc_timers_log_global_destroy(void); +#endif /* at least one profiler requested. */ #ifdef __cplusplus } diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index 73b3670ccc3..c44dcce7de3 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -198,7 +198,7 @@ grpc_channel *grpc_channel_create(const char *target, #define MAX_FILTERS 3 const grpc_channel_filter *filters[MAX_FILTERS]; int n = 0; - GRPC_STAP_TIMING_NS_BEGIN(1); + GRPC_TIMER_BEGIN(PTAG_OTHER_BASE + 2, NULL); filters[n++] = &grpc_client_surface_filter; if (grpc_channel_args_is_census_enabled(args)) { filters[n++] = &grpc_client_census_filter; @@ -214,6 +214,6 @@ grpc_channel *grpc_channel_create(const char *target, grpc_client_setup_create_and_attach(grpc_channel_get_channel_stack(channel), args, mdctx, initiate_setup, done_setup, s); - GRPC_STAP_TIMING_NS_END(1); + GRPC_TIMER_END(PTAG_OTHER_BASE + 2, NULL); return channel; } diff --git a/src/core/surface/init.c b/src/core/surface/init.c index 5a119a47cc7..bfee28e5fc1 100644 --- a/src/core/surface/init.c +++ b/src/core/surface/init.c @@ -64,7 +64,7 @@ void grpc_init(void) { grpc_iomgr_init(); grpc_tracer_init("GRPC_TRACE"); census_init(); - grpc_timers_log_global_init(); + grpc_timers_global_init(); } gpr_mu_unlock(&g_init_mu); } @@ -74,7 +74,7 @@ void grpc_shutdown(void) { if (--g_initializations == 0) { grpc_iomgr_shutdown(); census_shutdown(); - grpc_timers_log_global_destroy(); + grpc_timers_global_destroy(); } gpr_mu_unlock(&g_init_mu); } diff --git a/src/cpp/client/channel.cc b/src/cpp/client/channel.cc index c541ddfb487..3d9d1307a46 100644 --- a/src/cpp/client/channel.cc +++ b/src/cpp/client/channel.cc @@ -70,7 +70,7 @@ Call Channel::CreateCall(const RpcMethod& method, ClientContext* context, ? target_.c_str() : context->authority().c_str(), context->raw_deadline()); - GRPC_TIMER_MARK(CALL_CREATED, c_call); + GRPC_TIMER_MARK(PTAG_CPP_CALL_CREATED, c_call); context->set_call(c_call, shared_from_this()); return Call(c_call, this, cq); } @@ -79,11 +79,11 @@ void Channel::PerformOpsOnCall(CallOpBuffer* buf, Call* call) { static const size_t MAX_OPS = 8; size_t nops = MAX_OPS; grpc_op ops[MAX_OPS]; - GRPC_TIMER_MARK(PERFORM_OPS_BEGIN, call->call()); + GRPC_TIMER_BEGIN(PTAG_CPP_PERFORM_OPS, call->call()); buf->FillOps(ops, &nops); GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_batch(call->call(), ops, nops, buf)); - GRPC_TIMER_MARK(PERFORM_OPS_END, call->call()); + GRPC_TIMER_END(PTAG_CPP_PERFORM_OPS, call->call()); } void* Channel::RegisterMethod(const char* method) { diff --git a/src/cpp/common/call.cc b/src/cpp/common/call.cc index 9878133331d..41d64276e2b 100644 --- a/src/cpp/common/call.cc +++ b/src/cpp/common/call.cc @@ -232,13 +232,13 @@ void CallOpBuffer::FillOps(grpc_op* ops, size_t* nops) { } if (send_message_ || send_message_buffer_) { if (send_message_) { - GRPC_TIMER_MARK(SER_PROTO_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_PROTO_SERIALIZE, 0); bool success = SerializeProto(*send_message_, &send_buf_); if (!success) { abort(); // TODO handle parse failure } - GRPC_TIMER_MARK(SER_PROTO_END, 0); + GRPC_TIMER_END(PTAG_PROTO_SERIALIZE, 0); } else { send_buf_ = send_message_buffer_->buffer(); } @@ -310,10 +310,10 @@ bool CallOpBuffer::FinalizeResult(void** tag, bool* status) { if (recv_buf_) { got_message = *status; if (recv_message_) { - GRPC_TIMER_MARK(DESER_PROTO_BEGIN, 0); + GRPC_TIMER_BEGIN(PTAG_PROTO_DESERIALIZE, 0); *status = *status && DeserializeProto(recv_buf_, recv_message_); grpc_byte_buffer_destroy(recv_buf_); - GRPC_TIMER_MARK(DESER_PROTO_END, 0); + GRPC_TIMER_END(PTAG_PROTO_DESERIALIZE, 0); } else { recv_message_buffer_->set_buffer(recv_buf_); } diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc index 1d39378595c..c84b9d8d84d 100644 --- a/src/cpp/server/server.cc +++ b/src/cpp/server/server.cc @@ -124,12 +124,12 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag { std::unique_ptr req; std::unique_ptr res; if (has_request_payload_) { - GRPC_TIMER_MARK(DESER_PROTO_BEGIN, call_.call()); + GRPC_TIMER_BEGIN(PTAG_PROTO_DESERIALIZE, call_.call()); req.reset(method_->AllocateRequestProto()); if (!DeserializeProto(request_payload_, req.get())) { abort(); // for now } - GRPC_TIMER_MARK(DESER_PROTO_END, call_.call()); + GRPC_TIMER_END(PTAG_PROTO_DESERIALIZE, call_.call()); } if (has_response_payload_) { res.reset(method_->AllocateResponseProto()); @@ -343,9 +343,9 @@ class Server::AsyncRequest GRPC_FINAL : public CompletionQueueTag { bool orig_status = *status; if (*status && request_) { if (payload_) { - GRPC_TIMER_MARK(DESER_PROTO_BEGIN, call_); + GRPC_TIMER_BEGIN(PTAG_PROTO_DESERIALIZE, call_); *status = DeserializeProto(payload_, request_); - GRPC_TIMER_MARK(DESER_PROTO_END, call_); + GRPC_TIMER_END(PTAG_PROTO_DESERIALIZE, call_); } else { *status = false; } diff --git a/templates/Makefile.template b/templates/Makefile.template index 8fe8854d678..cd5eee6e5c4 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -101,14 +101,23 @@ CPPFLAGS_opt = -O2 LDFLAGS_opt = DEFINES_opt = NDEBUG -VALID_CONFIG_latprof = 1 -CC_latprof = $(DEFAULT_CC) -CXX_latprof = $(DEFAULT_CXX) -LD_latprof = $(DEFAULT_CC) -LDXX_latprof = $(DEFAULT_CXX) -CPPFLAGS_latprof = -O2 -DGRPC_LATENCY_PROFILER -LDFLAGS_latprof = -DEFINES_latprof = NDEBUG +VALID_CONFIG_basicprof = 1 +CC_basicprof = $(DEFAULT_CC) +CXX_basicprof = $(DEFAULT_CXX) +LD_basicprof = $(DEFAULT_CC) +LDXX_basicprof = $(DEFAULT_CXX) +CPPFLAGS_basicprof = -O2 -DGRPC_BASIC_PROFILER +LDFLAGS_basicprof = +DEFINES_basicprof = NDEBUG + +VALID_CONFIG_stapprof = 1 +CC_stapprof = $(DEFAULT_CC) +CXX_stapprof = $(DEFAULT_CXX) +LD_stapprof = $(DEFAULT_CC) +LDXX_stapprof = $(DEFAULT_CXX) +CPPFLAGS_stapprof = -O2 -DGRPC_STAP_PROFILER +LDFLAGS_stapprof = +DEFINES_stapprof = NDEBUG VALID_CONFIG_dbg = 1 CC_dbg = $(DEFAULT_CC) @@ -408,7 +417,6 @@ HAS_SYSTEMTAP = false ifeq ($(HAS_SYSTEMTAP_HEADERS),true) ifeq ($(HAS_DTRACE),true) HAS_SYSTEMTAP = true -DEFINES += GRPC_STAP_PROFILER endif endif @@ -591,6 +599,17 @@ protoc_dep_message: @echo " make run_dep_checks" @echo +systemtap_dep_error: + @echo + @echo "DEPENDENCY ERROR" + @echo + @echo "Under the '$(CONFIG)' configutation, the target you are trying " + @echo "to build requires systemtap 2.7+ (on Linux) or dtrace (on other " + @echo "platforms such as Solaris and *BSD). " + @echo + @echo "Please consult INSTALL to get more information." + @echo + stop: @false @@ -879,13 +898,15 @@ endif % endfor +ifeq ($(CONFIG), stapprof) ifeq ($(HAS_SYSTEMTAP),true) $(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d $(E) "[DTRACE] Compiling $<" $(Q) mkdir -p `dirname $@` $(Q) $(DTRACE) -C -h -s $< -o $@ - -src/core/profiling/timers.h: $(GENDIR)/src/core/profiling/stap_probes.h +else +$(GENDIR)/src/core/profiling/stap_probes.h: systemtap_dep_error stop +endif endif $(OBJDIR)/$(CONFIG)/%.o : %.c @@ -908,6 +929,17 @@ $(OBJDIR)/$(CONFIG)/%.o : %.cc $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< +ifeq ($(CONFIG), stapprof) +ifeq ($(HAS_SYSTEMTAP),true) +$(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d + $(E) "[DTRACE] Compiling $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(DTRACE) -C -h -s $< -o $@ +else +$(GENDIR)/src/core/profiling/stap_probes.c: systemtap_dep_error stop +endif +endif + install: install_c install_cxx install-plugins install-certs verify-install diff --git a/vsprojects/grpc/grpc.vcxproj b/vsprojects/grpc/grpc.vcxproj index fc7744f2b7b..83a44a37f6e 100644 --- a/vsprojects/grpc/grpc.vcxproj +++ b/vsprojects/grpc/grpc.vcxproj @@ -341,7 +341,9 @@ - + + + diff --git a/vsprojects/grpc/grpc.vcxproj.filters b/vsprojects/grpc/grpc.vcxproj.filters index 1dfca58cb55..c29355f6b10 100644 --- a/vsprojects/grpc/grpc.vcxproj.filters +++ b/vsprojects/grpc/grpc.vcxproj.filters @@ -217,7 +217,10 @@ src\core\json - + + src\core\profiling + + src\core\profiling diff --git a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj index 670b109e6fe..07debcccba8 100644 --- a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj @@ -286,7 +286,9 @@ - + + + diff --git a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters index 7c94d4d51e3..c886f87c88a 100644 --- a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -160,7 +160,10 @@ src\core\json - + + src\core\profiling + + src\core\profiling From c3edfc4b2b58b073fd9b981b1ab802a93bdc1877 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 10:41:16 -0700 Subject: [PATCH 05/17] Fixed repetitions in Makefile.template. --- Makefile | 12 ------------ templates/Makefile.template | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/Makefile b/Makefile index 73352dfad13..d0a1284eb0b 100644 --- a/Makefile +++ b/Makefile @@ -2845,18 +2845,6 @@ $(OBJDIR)/$(CONFIG)/%.o : %.cc $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< -ifeq ($(CONFIG), stapprof) -ifeq ($(HAS_SYSTEMTAP),true) -$(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d - $(E) "[DTRACE] Compiling $<" - $(Q) mkdir -p `dirname $@` - $(Q) $(DTRACE) -C -h -s $< -o $@ -else -$(GENDIR)/src/core/profiling/stap_probes.c: systemtap_dep_error stop -endif -endif - - install: install_c install_cxx install-plugins install-certs verify-install install_c: install-headers_c install-static_c install-shared_c diff --git a/templates/Makefile.template b/templates/Makefile.template index 86b22a9874a..fa1ff2b9bb2 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -929,18 +929,6 @@ $(OBJDIR)/$(CONFIG)/%.o : %.cc $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< -ifeq ($(CONFIG), stapprof) -ifeq ($(HAS_SYSTEMTAP),true) -$(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d - $(E) "[DTRACE] Compiling $<" - $(Q) mkdir -p `dirname $@` - $(Q) $(DTRACE) -C -h -s $< -o $@ -else -$(GENDIR)/src/core/profiling/stap_probes.c: systemtap_dep_error stop -endif -endif - - install: install_c install_cxx install-plugins install-certs verify-install install_c: install-headers_c install-static_c install-shared_c From 735987bea0e48cfe579b48a969d71298ae973c31 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 10:41:38 -0700 Subject: [PATCH 06/17] Added documentation to stap script. --- test/core/profiling/mark_timings.stp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/core/profiling/mark_timings.stp b/test/core/profiling/mark_timings.stp index a7ccc83c617..45d5e0d2174 100644 --- a/test/core/profiling/mark_timings.stp +++ b/test/core/profiling/mark_timings.stp @@ -1,10 +1,15 @@ -global starts, times, times_per_tag +/* This script requires a command line argument, to be used in the "process" + * probe definition. + * + * For a statically build binary, that'd be the name of the binary itself. + * For dinamically built ones, point to the location of the libgprc.so being + * used. */ -probe process.mark("timing_ns_begin") { +probe process(@1).mark("timing_ns_begin") { starts[$arg1, tid()] = gettimeofday_ns(); } -probe process.mark("timing_ns_end") { +probe process(@1).mark("timing_ns_end") { tag = $arg1 t = gettimeofday_ns(); if (s = starts[tag, tid()]) { @@ -27,6 +32,7 @@ probe end { } printf("%15s %10s %10s\n", "tag", "count", "avg(ns)"); foreach ([tag+] in times_per_tag) { - printf("%15X %10d %10d\n", tag, @count(times_per_tag[tag]), @avg(times_per_tag[tag])); + printf("%15X %10d %10d\n", tag, @count(times_per_tag[tag]), + @avg(times_per_tag[tag])); } } From bbc0b775e137e6865c9213086c61b84e7415a9ec Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 14:10:05 -0700 Subject: [PATCH 07/17] Merge comments. See below. - Added ptag ignore annotation threshold - prefixed everythig with grpc_ - removed spurious annotations in channel_create.c and client_channel.c - removed stap_probes.h (it's generated from stap_probes.d by make) - Updated Makefile and its template with the right way to generate stap probe headers from its .d definition. --- Makefile | 3 +- src/core/channel/client_channel.c | 4 --- src/core/iomgr/pollset_posix.c | 2 +- src/core/iomgr/tcp_posix.c | 20 ++++++------ src/core/profiling/basic_timers.c | 4 ++- src/core/profiling/stap_probes.h | 43 ------------------------- src/core/profiling/timers.h | 53 ++++++++++++++++++------------- src/core/surface/channel_create.c | 8 ++--- src/cpp/client/channel.cc | 6 ++-- src/cpp/common/call.cc | 8 ++--- src/cpp/server/server.cc | 8 ++--- templates/Makefile.template | 3 +- 12 files changed, 62 insertions(+), 100 deletions(-) delete mode 100644 src/core/profiling/stap_probes.h diff --git a/Makefile b/Makefile index d0a1284eb0b..1b6c44f7374 100644 --- a/Makefile +++ b/Makefile @@ -2814,7 +2814,8 @@ $(GENDIR)/test/proto/test.grpc.pb.cc: test/proto/test.proto $(PROTOBUF_DEP) $(PR endif -ifeq ($(CONFIG), stapprof) +ifeq ($(CONFIG),stapprof) +src/core/profiling/stap_timers.c: $(GENDIR)/src/core/profiling/stap_probes.h ifeq ($(HAS_SYSTEMTAP),true) $(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d $(E) "[DTRACE] Compiling $<" diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index 7be89e53497..bc481e59ca5 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -39,9 +39,7 @@ #include "src/core/channel/child_channel.h" #include "src/core/channel/connected_channel.h" #include "src/core/iomgr/iomgr.h" -#include "src/core/profiling/timers.h" #include "src/core/support/string.h" - #include #include #include @@ -129,7 +127,6 @@ static void complete_activate(grpc_call_element *elem, grpc_call_op *op) { static void start_rpc(grpc_call_element *elem, grpc_call_op *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - GRPC_TIMER_BEGIN(PTAG_OTHER_BASE + 1, NULL); gpr_mu_lock(&chand->mu); if (calld->state == CALL_CANCELLED) { gpr_mu_unlock(&chand->mu); @@ -175,7 +172,6 @@ static void start_rpc(grpc_call_element *elem, grpc_call_op *op) { grpc_transport_setup_initiate(chand->transport_setup); } } - GRPC_TIMER_BEGIN(PTAG_OTHER_BASE + 1, NULL); } static void remove_waiting_child(channel_data *chand, call_data *calld) { diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 4985dcf7399..4d1bcad9e27 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -411,7 +411,7 @@ static int unary_poll_pollset_maybe_work(grpc_pollset *pollset, pfd[1].events = grpc_fd_begin_poll(fd, pollset, POLLIN, POLLOUT, &fd_watcher); r = poll(pfd, GPR_ARRAY_SIZE(pfd), timeout); - GRPC_TIMER_MARK(PTAG_POLL_FINISHED, r); + GRPC_TIMER_MARK(GRPC_PTAG_POLL_FINISHED, r); grpc_fd_end_poll(&fd_watcher); diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index 7626a511846..40897fb8f86 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -327,7 +327,7 @@ static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { gpr_slice *final_slices; size_t final_nslices; - GRPC_TIMER_BEGIN(PTAG_HANDLE_READ, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_HANDLE_READ, 0); slice_state_init(&read_state, static_read_slices, INLINE_SLICE_BUFFER_SIZE, 0); @@ -350,11 +350,11 @@ static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { msg.msg_controllen = 0; msg.msg_flags = 0; - GRPC_TIMER_BEGIN(PTAG_RECVMSG, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_RECVMSG, 0); do { read_bytes = recvmsg(tcp->fd, &msg, 0); } while (read_bytes < 0 && errno == EINTR); - GRPC_TIMER_END(PTAG_RECVMSG, 0); + GRPC_TIMER_END(GRPC_PTAG_RECVMSG, 0); if (read_bytes < allocated_bytes) { /* TODO(klempner): Consider a second read first, in hopes of getting a @@ -406,7 +406,7 @@ static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { ++iov_size; } } - GRPC_TIMER_END(PTAG_HANDLE_READ, 0); + GRPC_TIMER_END(GRPC_PTAG_HANDLE_READ, 0); } static void grpc_tcp_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, @@ -438,12 +438,12 @@ static grpc_endpoint_write_status grpc_tcp_flush(grpc_tcp *tcp) { msg.msg_controllen = 0; msg.msg_flags = 0; - GRPC_TIMER_BEGIN(PTAG_SENDMSG, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_SENDMSG, 0); do { /* TODO(klempner): Cork if this is a partial write */ sent_length = sendmsg(tcp->fd, &msg, 0); } while (sent_length < 0 && errno == EINTR); - GRPC_TIMER_END(PTAG_SENDMSG, 0); + GRPC_TIMER_END(GRPC_PTAG_SENDMSG, 0); if (sent_length < 0) { if (errno == EAGAIN) { @@ -479,7 +479,7 @@ static void grpc_tcp_handle_write(void *arg /* grpc_tcp */, int success) { return; } - GRPC_TIMER_BEGIN(PTAG_TCP_CB_WRITE, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_TCP_CB_WRITE, 0); write_status = grpc_tcp_flush(tcp); if (write_status == GRPC_ENDPOINT_WRITE_PENDING) { grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_closure); @@ -495,7 +495,7 @@ static void grpc_tcp_handle_write(void *arg /* grpc_tcp */, int success) { cb(tcp->write_user_data, cb_status); grpc_tcp_unref(tcp); } - GRPC_TIMER_END(PTAG_TCP_CB_WRITE, 0); + GRPC_TIMER_END(GRPC_PTAG_TCP_CB_WRITE, 0); } static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, @@ -518,7 +518,7 @@ static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, } } - GRPC_TIMER_BEGIN(PTAG_TCP_WRITE, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_TCP_WRITE, 0); GPR_ASSERT(tcp->write_cb == NULL); slice_state_init(&tcp->write_state, slices, nslices, nslices); @@ -532,7 +532,7 @@ static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_closure); } - GRPC_TIMER_END(PTAG_TCP_WRITE, 0); + GRPC_TIMER_END(GRPC_PTAG_TCP_WRITE, 0); return status; } diff --git a/src/core/profiling/basic_timers.c b/src/core/profiling/basic_timers.c index b33afa7e8cc..f290875f781 100644 --- a/src/core/profiling/basic_timers.c +++ b/src/core/profiling/basic_timers.c @@ -129,7 +129,9 @@ static void grpc_timers_log_add(grpc_timers_log* log, int tag, void* id, /* Latency profiler API implementation. */ void grpc_timer_add_mark(int tag, void* id, const char* file, int line) { - grpc_timers_log_add(grpc_timers_log_global, tag, id, file, line); + if (tag <= GRPC_PTAG_IGNORE_THRESHOLD) { + grpc_timers_log_add(grpc_timers_log_global, tag, id, file, line); + } } void grpc_timer_begin(int tag, void* id, const char *file, int line) {} diff --git a/src/core/profiling/stap_probes.h b/src/core/profiling/stap_probes.h deleted file mode 100644 index ba8020c5a32..00000000000 --- a/src/core/profiling/stap_probes.h +++ /dev/null @@ -1,43 +0,0 @@ -/* Generated by the Systemtap dtrace wrapper */ - -#define _SDT_HAS_SEMAPHORES 1 - - -#define STAP_HAS_SEMAPHORES 1 /* deprecated */ - - -#include - -/* _STAP_ADD_MARK ( int tag) */ -#if defined STAP_SDT_V1 -#define _STAP_ADD_MARK_ENABLED() __builtin_expect (add_mark_semaphore, 0) -#define _stap_add_mark_semaphore add_mark_semaphore -#else -#define _STAP_ADD_MARK_ENABLED() __builtin_expect (_stap_add_mark_semaphore, 0) -#endif -__extension__ extern unsigned short _stap_add_mark_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); -#define _STAP_ADD_MARK(arg1) \ -DTRACE_PROBE1(_stap,add_mark,arg1) - -/* _STAP_TIMING_NS_BEGIN ( int tag) */ -#if defined STAP_SDT_V1 -#define _STAP_TIMING_NS_BEGIN_ENABLED() __builtin_expect (timing_ns_begin_semaphore, 0) -#define _stap_timing_ns_begin_semaphore timing_ns_begin_semaphore -#else -#define _STAP_TIMING_NS_BEGIN_ENABLED() __builtin_expect (_stap_timing_ns_begin_semaphore, 0) -#endif -__extension__ extern unsigned short _stap_timing_ns_begin_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); -#define _STAP_TIMING_NS_BEGIN(arg1) \ -DTRACE_PROBE1(_stap,timing_ns_begin,arg1) - -/* _STAP_TIMING_NS_END ( int tag) */ -#if defined STAP_SDT_V1 -#define _STAP_TIMING_NS_END_ENABLED() __builtin_expect (timing_ns_end_semaphore, 0) -#define _stap_timing_ns_end_semaphore timing_ns_end_semaphore -#else -#define _STAP_TIMING_NS_END_ENABLED() __builtin_expect (_stap_timing_ns_end_semaphore, 0) -#endif -__extension__ extern unsigned short _stap_timing_ns_end_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); -#define _STAP_TIMING_NS_END(arg1) \ -DTRACE_PROBE1(_stap,timing_ns_end,arg1) - diff --git a/src/core/profiling/timers.h b/src/core/profiling/timers.h index a78bbea4ec8..d0b8286c03e 100644 --- a/src/core/profiling/timers.h +++ b/src/core/profiling/timers.h @@ -45,27 +45,30 @@ void grpc_timer_add_mark(int tag, void* id, const char *file, int line); void grpc_timer_begin(int tag, void* id, const char *file, int line); void grpc_timer_end(int tag, void* id, const char *file, int line); -enum profiling_tags { +enum grpc_profiling_tags { + /* Any GRPC_PTAG_* >= than the threshold won't generate any profiling mark. */ + GRPC_PTAG_IGNORE_THRESHOLD = 1000000, + /* Re. Protos. */ - PTAG_PROTO_SERIALIZE = 100, - PTAG_PROTO_DESERIALIZE = 101, + GRPC_PTAG_PROTO_SERIALIZE = 100 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_PROTO_DESERIALIZE = 101 + GRPC_PTAG_IGNORE_THRESHOLD, /* Re. sockets. */ - PTAG_HANDLE_READ = 200, - PTAG_SENDMSG = 201, - PTAG_RECVMSG = 202, - PTAG_POLL_FINISHED = 203, - PTAG_TCP_CB_WRITE = 204, - PTAG_TCP_WRITE = 205, + GRPC_PTAG_HANDLE_READ = 200 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_SENDMSG = 201 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_RECVMSG = 202 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_POLL_FINISHED = 203 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_TCP_CB_WRITE = 204 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_TCP_WRITE = 205 + GRPC_PTAG_IGNORE_THRESHOLD, /* C++ */ - PTAG_CPP_CALL_CREATED = 300, - PTAG_CPP_PERFORM_OPS = 301, + GRPC_PTAG_CPP_CALL_CREATED = 300 + GRPC_PTAG_IGNORE_THRESHOLD, + GRPC_PTAG_CPP_PERFORM_OPS = 301 + GRPC_PTAG_IGNORE_THRESHOLD, /* > 1024 Unassigned reserved. For any miscellaneous use. - * Use addition to generate tags from this base or take advantage of the 10 - * zero'd bits for OR-ing. */ - PTAG_OTHER_BASE = 1024 + * Use addition to generate tags from this base or take advantage of the 10 + * zero'd bits for OR-ing. */ + GRPC_PTAG_OTHER_BASE = 1024 }; #if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER)) @@ -86,14 +89,20 @@ enum profiling_tags { #endif /* Generic profiling interface. */ -#define GRPC_TIMER_MARK(tag, id) \ - grpc_timer_add_mark(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__) - -#define GRPC_TIMER_BEGIN(tag, id) \ - grpc_timer_begin(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__) - -#define GRPC_TIMER_END(tag, id) \ - grpc_timer_end(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__) +#define GRPC_TIMER_MARK(tag, id) \ + if (tag < GRPC_PTAG_IGNORE_THRESHOLD) { \ + grpc_timer_add_mark(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__); \ + } + +#define GRPC_TIMER_BEGIN(tag, id) \ + if (tag < GRPC_PTAG_IGNORE_THRESHOLD) { \ + grpc_timer_begin(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__); \ + } + +#define GRPC_TIMER_END(tag, id) \ + if (tag < GRPC_PTAG_IGNORE_THRESHOLD) { \ + grpc_timer_end(tag, ((void *)(gpr_intptr)(id)), __FILE__, __LINE__); \ + } #ifdef GRPC_STAP_PROFILER /* Empty placeholder for now. */ diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index c44dcce7de3..3104b1d00d5 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -31,7 +31,6 @@ * */ - #include "src/core/iomgr/sockaddr.h" #include @@ -49,12 +48,10 @@ #include "src/core/iomgr/endpoint.h" #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" -#include "src/core/profiling/timers.h" -#include "src/core/support/string.h" #include "src/core/surface/channel.h" #include "src/core/surface/client.h" +#include "src/core/support/string.h" #include "src/core/transport/chttp2_transport.h" - #include #include #include @@ -198,7 +195,6 @@ grpc_channel *grpc_channel_create(const char *target, #define MAX_FILTERS 3 const grpc_channel_filter *filters[MAX_FILTERS]; int n = 0; - GRPC_TIMER_BEGIN(PTAG_OTHER_BASE + 2, NULL); filters[n++] = &grpc_client_surface_filter; if (grpc_channel_args_is_census_enabled(args)) { filters[n++] = &grpc_client_census_filter; @@ -214,6 +210,6 @@ grpc_channel *grpc_channel_create(const char *target, grpc_client_setup_create_and_attach(grpc_channel_get_channel_stack(channel), args, mdctx, initiate_setup, done_setup, s); - GRPC_TIMER_END(PTAG_OTHER_BASE + 2, NULL); + return channel; } diff --git a/src/cpp/client/channel.cc b/src/cpp/client/channel.cc index 3d9d1307a46..475a20d8832 100644 --- a/src/cpp/client/channel.cc +++ b/src/cpp/client/channel.cc @@ -70,7 +70,7 @@ Call Channel::CreateCall(const RpcMethod& method, ClientContext* context, ? target_.c_str() : context->authority().c_str(), context->raw_deadline()); - GRPC_TIMER_MARK(PTAG_CPP_CALL_CREATED, c_call); + GRPC_TIMER_MARK(GRPC_PTAG_CPP_CALL_CREATED, c_call); context->set_call(c_call, shared_from_this()); return Call(c_call, this, cq); } @@ -79,11 +79,11 @@ void Channel::PerformOpsOnCall(CallOpBuffer* buf, Call* call) { static const size_t MAX_OPS = 8; size_t nops = MAX_OPS; grpc_op ops[MAX_OPS]; - GRPC_TIMER_BEGIN(PTAG_CPP_PERFORM_OPS, call->call()); + GRPC_TIMER_BEGIN(GRPC_PTAG_CPP_PERFORM_OPS, call->call()); buf->FillOps(ops, &nops); GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_batch(call->call(), ops, nops, buf)); - GRPC_TIMER_END(PTAG_CPP_PERFORM_OPS, call->call()); + GRPC_TIMER_END(GRPC_PTAG_CPP_PERFORM_OPS, call->call()); } void* Channel::RegisterMethod(const char* method) { diff --git a/src/cpp/common/call.cc b/src/cpp/common/call.cc index 41d64276e2b..a60282316ae 100644 --- a/src/cpp/common/call.cc +++ b/src/cpp/common/call.cc @@ -232,13 +232,13 @@ void CallOpBuffer::FillOps(grpc_op* ops, size_t* nops) { } if (send_message_ || send_message_buffer_) { if (send_message_) { - GRPC_TIMER_BEGIN(PTAG_PROTO_SERIALIZE, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_SERIALIZE, 0); bool success = SerializeProto(*send_message_, &send_buf_); if (!success) { abort(); // TODO handle parse failure } - GRPC_TIMER_END(PTAG_PROTO_SERIALIZE, 0); + GRPC_TIMER_END(GRPC_PTAG_PROTO_SERIALIZE, 0); } else { send_buf_ = send_message_buffer_->buffer(); } @@ -310,10 +310,10 @@ bool CallOpBuffer::FinalizeResult(void** tag, bool* status) { if (recv_buf_) { got_message = *status; if (recv_message_) { - GRPC_TIMER_BEGIN(PTAG_PROTO_DESERIALIZE, 0); + GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_DESERIALIZE, 0); *status = *status && DeserializeProto(recv_buf_, recv_message_); grpc_byte_buffer_destroy(recv_buf_); - GRPC_TIMER_END(PTAG_PROTO_DESERIALIZE, 0); + GRPC_TIMER_END(GRPC_PTAG_PROTO_DESERIALIZE, 0); } else { recv_message_buffer_->set_buffer(recv_buf_); } diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc index de536ef2cf3..34aac4c5845 100644 --- a/src/cpp/server/server.cc +++ b/src/cpp/server/server.cc @@ -124,12 +124,12 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag { std::unique_ptr req; std::unique_ptr res; if (has_request_payload_) { - GRPC_TIMER_BEGIN(PTAG_PROTO_DESERIALIZE, call_.call()); + GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_DESERIALIZE, call_.call()); req.reset(method_->AllocateRequestProto()); if (!DeserializeProto(request_payload_, req.get())) { abort(); // for now } - GRPC_TIMER_END(PTAG_PROTO_DESERIALIZE, call_.call()); + GRPC_TIMER_END(GRPC_PTAG_PROTO_DESERIALIZE, call_.call()); } if (has_response_payload_) { res.reset(method_->AllocateResponseProto()); @@ -346,9 +346,9 @@ class Server::AsyncRequest GRPC_FINAL : public CompletionQueueTag { bool orig_status = *status; if (*status && request_) { if (payload_) { - GRPC_TIMER_BEGIN(PTAG_PROTO_DESERIALIZE, call_); + GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_DESERIALIZE, call_); *status = DeserializeProto(payload_, request_); - GRPC_TIMER_END(PTAG_PROTO_DESERIALIZE, call_); + GRPC_TIMER_END(GRPC_PTAG_PROTO_DESERIALIZE, call_); } else { *status = false; } diff --git a/templates/Makefile.template b/templates/Makefile.template index fa1ff2b9bb2..4c63fb2ed95 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -898,7 +898,8 @@ endif % endfor -ifeq ($(CONFIG), stapprof) +ifeq ($(CONFIG),stapprof) +src/core/profiling/stap_timers.c: $(GENDIR)/src/core/profiling/stap_probes.h ifeq ($(HAS_SYSTEMTAP),true) $(GENDIR)/src/core/profiling/stap_probes.h: src/core/profiling/stap_probes.d $(E) "[DTRACE] Compiling $<" From b220c1d21342fb995881c39efb6b4b8c55e89e6b Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 14:26:08 -0700 Subject: [PATCH 08/17] Added missing globals declaration to STAP script. --- test/core/profiling/mark_timings.stp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/core/profiling/mark_timings.stp b/test/core/profiling/mark_timings.stp index 45d5e0d2174..0c0a417faf7 100644 --- a/test/core/profiling/mark_timings.stp +++ b/test/core/profiling/mark_timings.stp @@ -5,6 +5,8 @@ * For dinamically built ones, point to the location of the libgprc.so being * used. */ +global starts, times, times_per_tag + probe process(@1).mark("timing_ns_begin") { starts[$arg1, tid()] = gettimeofday_ns(); } From 7af45f2821504f19244140d4ea9bf69e5d6bf67c Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 16:22:09 -0700 Subject: [PATCH 09/17] Fixed bad merge on basic_timers.c --- src/core/profiling/basic_timers.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/profiling/basic_timers.c b/src/core/profiling/basic_timers.c index 7ad67f4831f..8dd944cd366 100644 --- a/src/core/profiling/basic_timers.c +++ b/src/core/profiling/basic_timers.c @@ -47,7 +47,6 @@ typedef struct grpc_timer_entry { grpc_precise_clock tm; - int tag; gpr_thd_id thd; int tag; void* id; From 31e3a6fc69aea5b5af34d86783866d113f9169c3 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 29 Apr 2015 16:36:25 -0700 Subject: [PATCH 10/17] Removed redundant if from basic_timers.c --- src/core/profiling/basic_timers.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/core/profiling/basic_timers.c b/src/core/profiling/basic_timers.c index 8dd944cd366..d89bba7b878 100644 --- a/src/core/profiling/basic_timers.c +++ b/src/core/profiling/basic_timers.c @@ -132,9 +132,7 @@ static void grpc_timers_log_add(grpc_timers_log* log, int tag, void* id, /* Latency profiler API implementation. */ void grpc_timer_add_mark(int tag, void* id, const char* file, int line) { - if (tag <= GRPC_PTAG_IGNORE_THRESHOLD) { - grpc_timers_log_add(grpc_timers_log_global, tag, id, file, line); - } + grpc_timers_log_add(grpc_timers_log_global, tag, id, file, line); } void grpc_timer_begin(int tag, void* id, const char *file, int line) {} From 0e67cb2c3958097786d2391c6da86657ceedc4e4 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 30 Apr 2015 09:03:00 -0700 Subject: [PATCH 11/17] Fixed missing name change from timers.h refactor. --- test/core/profiling/timers_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/profiling/timers_test.c b/test/core/profiling/timers_test.c index 55e59c969e8..12b08c115e5 100644 --- a/test/core/profiling/timers_test.c +++ b/test/core/profiling/timers_test.c @@ -76,8 +76,8 @@ void test_log_events(int num_seqs) { int main(int argc, char **argv) { grpc_test_init(argc, argv); - grpc_timers_log_global_init(); + grpc_timers_global_init(); test_log_events(1000000); - grpc_timers_log_global_destroy(); + grpc_timers_global_destroy(); return 0; } From 3b1e37d16df885474b5c1b7cc2728bfe6ab3a62e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Apr 2015 09:12:31 -0700 Subject: [PATCH 12/17] Fix errant empty data frame after trailers With the metadata batching changes recently, I introduced a bug whereby the http2 stream encoder would end up not being able to set the close bit on a header frame, and instead placed it on a new data frame, which is illegal by the HTTP2 spec. --- src/core/transport/chttp2/stream_encoder.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/stream_encoder.c index cf1e66bf8be..cf78ac50cc5 100644 --- a/src/core/transport/chttp2/stream_encoder.c +++ b/src/core/transport/chttp2/stream_encoder.c @@ -122,6 +122,12 @@ static void begin_frame(framer_state *st, frame_type type) { st->output_length_at_start_of_frame = st->output->length; } +static void begin_new_frame(framer_state *st, frame_type type) { + finish_frame(st, 1, 0); + st->last_was_header = 0; + begin_frame(st, type); +} + /* make sure that the current frame is of the type desired, and has sufficient space to add at least about_to_add bytes -- finishes the current frame if needed */ @@ -571,6 +577,7 @@ void grpc_chttp2_encode(grpc_stream_op *ops, size_t ops_count, int eof, a metadata element that needs to be unreffed back into the metadata slot. THIS MAY NOT BE THE SAME ELEMENT (if a decoder table slot got updated). After this loop, we'll do a batch unref of elements. */ + begin_new_frame(&st, HEADER); need_unref |= op->data.metadata.garbage.head != NULL; grpc_metadata_batch_assert_ok(&op->data.metadata); for (l = op->data.metadata.list.head; l; l = l->next) { @@ -580,9 +587,6 @@ void grpc_chttp2_encode(grpc_stream_op *ops, size_t ops_count, int eof, if (gpr_time_cmp(op->data.metadata.deadline, gpr_inf_future) != 0) { deadline_enc(compressor, op->data.metadata.deadline, &st); } - ensure_frame_type(&st, HEADER, 0); - finish_frame(&st, 1, 0); - st.last_was_header = 0; /* force a new header frame */ curop++; break; case GRPC_OP_SLICE: From 42b14cba51f85dfa578034d2b9eff16a94bf0d61 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Apr 2015 09:21:42 -0700 Subject: [PATCH 13/17] Remove dbg build from Travis, use gcov instead --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82d8316dda3..1f96c6fc006 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,13 +18,12 @@ env: - NUGET="mono nuget.exe" matrix: - CONFIG=opt TEST=sanity - - CONFIG=dbg TEST="c c++" + - CONFIG=gcov TEST="c c++" - CONFIG=opt TEST="c c++" - CONFIG=opt TEST=node - CONFIG=opt TEST=ruby - CONFIG=opt TEST=python - CONFIG=opt TEST=csharp - - CONFIG=gcov TEST="c c++" - USE_GCC=4.4 CONFIG=opt TEST=build - USE_GCC=4.5 CONFIG=opt TEST=build script: From 2e4a7dc2fee24be341be0e84c2136aef4f426991 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Apr 2015 09:22:31 -0700 Subject: [PATCH 14/17] Make gcov compile flags reflect dbg --- Makefile | 2 +- templates/Makefile.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d67ab17c614..991bed3b428 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ LD_gcov = gcc LDXX_gcov = g++ CPPFLAGS_gcov = -O0 -fprofile-arcs -ftest-coverage LDFLAGS_gcov = -fprofile-arcs -ftest-coverage -DEFINES_gcov = NDEBUG +DEFINES_gcov = _DEBUG DEBUG # General settings. diff --git a/templates/Makefile.template b/templates/Makefile.template index fe455d1c208..9a12cf99226 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -188,7 +188,7 @@ LD_gcov = gcc LDXX_gcov = g++ CPPFLAGS_gcov = -O0 -fprofile-arcs -ftest-coverage LDFLAGS_gcov = -fprofile-arcs -ftest-coverage -DEFINES_gcov = NDEBUG +DEFINES_gcov = _DEBUG DEBUG # General settings. From 0e08aed61c022f84ab80306565bf71b8e8613ff0 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 30 Apr 2015 09:25:45 -0700 Subject: [PATCH 15/17] add php service account credentials auth interop test, tested against cloud server --- src/php/composer.json | 3 +- src/php/composer.lock | 302 ++++++++++++++++++++++- src/php/tests/interop/interop_client.php | 77 +++++- 3 files changed, 366 insertions(+), 16 deletions(-) diff --git a/src/php/composer.json b/src/php/composer.json index 3c0cb37231f..ba7a1302f27 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -5,7 +5,8 @@ "homepage": "http://grpc.io", "license": "BSD-3-Clause", "require": { - "php": ">=5.5.0" + "php": ">=5.5.0", + "google/auth": "dev-master" }, "autoload": { "psr-4": { diff --git a/src/php/composer.lock b/src/php/composer.lock index 47fc70fd2d1..c2d723def45 100644 --- a/src/php/composer.lock +++ b/src/php/composer.lock @@ -4,12 +4,308 @@ "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "65467a098f5fd8b8fe5f7f6e10226f8a", - "packages": [], + "hash": "bb81ea5f72ddea2f594a172ff0f3b44d", + "packages": [ + { + "name": "firebase/php-jwt", + "version": "2.0.0", + "target-dir": "Firebase/PHP-JWT", + "source": { + "type": "git", + "url": "https://github.com/firebase/php-jwt.git", + "reference": "ffcfd888ce1e4f2d70cac2dc9b7301038332fe57" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/firebase/php-jwt/zipball/ffcfd888ce1e4f2d70cac2dc9b7301038332fe57", + "reference": "ffcfd888ce1e4f2d70cac2dc9b7301038332fe57", + "shasum": "" + }, + "require": { + "php": ">=5.2.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "Authentication/", + "Exceptions/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Neuman Vong", + "email": "neuman+pear@twilio.com", + "role": "Developer" + }, + { + "name": "Anant Narayanan", + "email": "anant@php.net", + "role": "Developer" + } + ], + "description": "A simple library to encode and decode JSON Web Tokens (JWT) in PHP. Should conform to the current spec.", + "homepage": "https://github.com/firebase/php-jwt", + "time": "2015-04-01 18:46:38" + }, + { + "name": "google/auth", + "version": "dev-master", + "source": { + "type": "git", + "url": "https://github.com/google/google-auth-library-php.git", + "reference": "35f87159b327fa6416266948c1747c585a4ae3ad" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/google/google-auth-library-php/zipball/35f87159b327fa6416266948c1747c585a4ae3ad", + "reference": "35f87159b327fa6416266948c1747c585a4ae3ad", + "shasum": "" + }, + "require": { + "firebase/php-jwt": "2.0.0", + "guzzlehttp/guzzle": "5.2.*", + "php": ">=5.4" + }, + "require-dev": { + "phplint/phplint": "0.0.1", + "phpunit/phpunit": "3.7.*" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ], + "psr-4": { + "Google\\Auth\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Google Auth Library for PHP", + "homepage": "http://github.com/google/google-auth-library-php", + "keywords": [ + "Authentication", + "google", + "oauth2" + ], + "time": "2015-04-30 11:57:19" + }, + { + "name": "guzzlehttp/guzzle", + "version": "5.2.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/guzzle.git", + "reference": "475b29ccd411f2fa8a408e64576418728c032cfa" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/475b29ccd411f2fa8a408e64576418728c032cfa", + "reference": "475b29ccd411f2fa8a408e64576418728c032cfa", + "shasum": "" + }, + "require": { + "guzzlehttp/ringphp": "~1.0", + "php": ">=5.4.0" + }, + "require-dev": { + "ext-curl": "*", + "phpunit/phpunit": "~4.0", + "psr/log": "~1.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.0-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Guzzle is a PHP HTTP client library and framework for building RESTful web service clients", + "homepage": "http://guzzlephp.org/", + "keywords": [ + "client", + "curl", + "framework", + "http", + "http client", + "rest", + "web service" + ], + "time": "2015-01-28 01:03:29" + }, + { + "name": "guzzlehttp/ringphp", + "version": "1.0.7", + "source": { + "type": "git", + "url": "https://github.com/guzzle/RingPHP.git", + "reference": "52d868f13570a9a56e5fce6614e0ec75d0f13ac2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/52d868f13570a9a56e5fce6614e0ec75d0f13ac2", + "reference": "52d868f13570a9a56e5fce6614e0ec75d0f13ac2", + "shasum": "" + }, + "require": { + "guzzlehttp/streams": "~3.0", + "php": ">=5.4.0", + "react/promise": "~2.0" + }, + "require-dev": { + "ext-curl": "*", + "phpunit/phpunit": "~4.0" + }, + "suggest": { + "ext-curl": "Guzzle will use specific adapters if cURL is present" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Ring\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", + "time": "2015-03-30 01:43:20" + }, + { + "name": "guzzlehttp/streams", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/streams.git", + "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/streams/zipball/47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", + "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", + "shasum": "" + }, + "require": { + "php": ">=5.4.0" + }, + "require-dev": { + "phpunit/phpunit": "~4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Stream\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Provides a simple abstraction over streams of data", + "homepage": "http://guzzlephp.org/", + "keywords": [ + "Guzzle", + "stream" + ], + "time": "2014-10-12 19:18:40" + }, + { + "name": "react/promise", + "version": "v2.2.0", + "source": { + "type": "git", + "url": "https://github.com/reactphp/promise.git", + "reference": "365fcee430dfa4ace1fbc75737ca60ceea7eeeef" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/promise/zipball/365fcee430dfa4ace1fbc75737ca60ceea7eeeef", + "reference": "365fcee430dfa4ace1fbc75737ca60ceea7eeeef", + "shasum": "" + }, + "require": { + "php": ">=5.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "psr-4": { + "React\\Promise\\": "src/" + }, + "files": [ + "src/functions_include.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jan Sorgalla", + "email": "jsorgalla@googlemail.com" + } + ], + "description": "A lightweight implementation of CommonJS Promises/A for PHP", + "time": "2014-12-30 13:32:42" + } + ], "packages-dev": [], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "google/auth": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index 6f81bfa6cd5..2c1c13caa1d 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -45,7 +45,7 @@ require 'test.php'; * @param $error_message Message to display if the assertion is false */ function hardAssert($value, $error_message) { - if(!$value) { + if (!$value) { echo $error_message . "\n"; exit(1); } @@ -53,7 +53,7 @@ function hardAssert($value, $error_message) { /** * Run the empty_unary test. - * Currently not tested against any server as of 2014-12-04 + * Passes when run against the Node server as of 2015-04-30 * @param $stub Stub object that has service methods */ function emptyUnary($stub) { @@ -64,11 +64,20 @@ function emptyUnary($stub) { /** * Run the large_unary test. - * Passes when run against the C++ server as of 2014-12-04 - * Not tested against any other server as of 2014-12-04 + * Passes when run against the C++/Node server as of 2015-04-30 * @param $stub Stub object that has service methods */ function largeUnary($stub) { + performLargeUnary($stub); +} + +/** + * Shared code between large unary test and auth test + * @param $stub Stub object that has service methods + * @param $fillUsername boolean whether to fill result with username + * @param $fillOauthScope boolean whether to fill result with oauth scope + */ +function performLargeUnary($stub, $fillUsername = false, $fillOauthScope = false) { $request_len = 271828; $response_len = 314159; @@ -79,6 +88,8 @@ function largeUnary($stub) { $payload->setType(grpc\testing\PayloadType::COMPRESSABLE); $payload->setBody(str_repeat("\0", $request_len)); $request->setPayload($payload); + $request->setFillUsername($fillUsername); + $request->setFillOauthScope($fillOauthScope); list($result, $status) = $stub->UnaryCall($request)->wait(); hardAssert($status->code === Grpc\STATUS_OK, 'Call did not complete successfully'); @@ -90,6 +101,27 @@ function largeUnary($stub) { 'Payload had the wrong length'); hardAssert($payload->getBody() === str_repeat("\0", $response_len), 'Payload had the wrong content'); + return $result; +} + +/** + * Run the service account credentials auth test. + * Passes when run against the cloud server as of 2015-04-30 + * @param $stub Stub object that has service methods + * @param $args array command line args + */ +function serviceAccountCreds($stub, $args) { + if (!array_key_exists('oauth_scope', $args)) { + throw new Exception('Missing oauth scope'); + } + $jsonKey = json_decode( + file_get_contents(getenv(Google\Auth\CredentialsLoader::ENV_VAR)), + true); + $result = performLargeUnary($stub, $fillUsername=true, $fillOauthScope=true); + hardAssert($result->getUsername() == $jsonKey['client_email'], + 'invalid email returned'); + hardAssert(strpos($args['oauth_scope'], $result->getOauthScope()) !== false, + 'invalid oauth scope returned'); } /** @@ -201,7 +233,8 @@ function cancelAfterFirstResponse($stub) { 'Call status was not CANCELLED'); } -$args = getopt('', array('server_host:', 'server_port:', 'test_case:')); +$args = getopt('', array('server_host:', 'server_port:', 'test_case:', + 'server_host_override:', 'oauth_scope:')); if (!array_key_exists('server_host', $args) || !array_key_exists('server_port', $args) || !array_key_exists('test_case', $args)) { @@ -210,20 +243,37 @@ if (!array_key_exists('server_host', $args) || $server_address = $args['server_host'] . ':' . $args['server_port']; -$credentials = Grpc\Credentials::createSsl( - file_get_contents(dirname(__FILE__) . '/../data/ca.pem')); +if (!array_key_exists('server_host_override', $args)) { + $args['server_host_override'] = 'foo.test.google.fr'; +} + +$ssl_cert_file = getenv('SSL_CERT_FILE'); +if (!$ssl_cert_file) { + $ssl_cert_file = dirname(__FILE__) . '/../data/ca.pem'; +} + +$credentials = Grpc\Credentials::createSsl(file_get_contents($ssl_cert_file)); + +$opts = [ + 'grpc.ssl_target_name_override' => $args['server_host_override'], + 'credentials' => $credentials, + ]; + +if (array_key_exists('oauth_scope', $args)) { + $auth = Google\Auth\ApplicationDefaultCredentials::getCredentials( + $args['oauth_scope']); + $opts['update_metadata'] = $auth->getUpdateMetadataFunc(); +} + $stub = new grpc\testing\TestServiceClient( new Grpc\BaseStub( $server_address, - [ - 'grpc.ssl_target_name_override' => 'foo.test.google.fr', - 'credentials' => $credentials - ])); + $opts)); echo "Connecting to $server_address\n"; echo "Running test case $args[test_case]\n"; -switch($args['test_case']) { +switch ($args['test_case']) { case 'empty_unary': emptyUnary($stub); break; @@ -242,6 +292,9 @@ switch($args['test_case']) { case 'cancel_after_first_response': cancelAfterFirstResponse($stub); break; + case 'service_account_creds': + serviceAccountCreds($stub, $args); + break; default: exit(1); } From 4fbc53e82d11a66f8570e71c65a087f210fd10ca Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 30 Apr 2015 09:37:07 -0700 Subject: [PATCH 16/17] more comments --- src/php/tests/interop/interop_client.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index 2c1c13caa1d..22f85aa322d 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -38,6 +38,7 @@ require 'empty.php'; require 'message_set.php'; require 'messages.php'; require 'test.php'; + /** * Assertion function that always exits with an error code if the assertion is * falsy @@ -126,7 +127,7 @@ function serviceAccountCreds($stub, $args) { /** * Run the client_streaming test. - * Not tested against any server as of 2014-12-04. + * Passes when run against the Node server as of 2015-04-30 * @param $stub Stub object that has service methods */ function clientStreaming($stub) { @@ -149,7 +150,7 @@ function clientStreaming($stub) { /** * Run the server_streaming test. - * Not tested against any server as of 2014-12-04. + * Passes when run against the Node server as of 2015-04-30 * @param $stub Stub object that has service methods. */ function serverStreaming($stub) { @@ -180,7 +181,7 @@ function serverStreaming($stub) { /** * Run the ping_pong test. - * Not tested against any server as of 2014-12-04. + * Passes when run against the Node server as of 2015-04-30 * @param $stub Stub object that has service methods. */ function pingPong($stub) { @@ -214,6 +215,11 @@ function pingPong($stub) { 'Call did not complete successfully'); } +/** + * Run the cancel_after_first_response test. + * Passes when run against the Node server as of 2015-04-30 + * @param $stub Stub object that has service methods. + */ function cancelAfterFirstResponse($stub) { $call = $stub->FullDuplexCall(); $request = new grpc\testing\StreamingOutputCallRequest(); From fae9d99e6a2671d5628402a3789c6800001cb25b Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 30 Apr 2015 12:46:26 -0700 Subject: [PATCH 17/17] Release Python 0.5.0a1 --- src/python/src/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/src/setup.py b/src/python/src/setup.py index 7149a9eb859..76750496776 100644 --- a/src/python/src/setup.py +++ b/src/python/src/setup.py @@ -86,7 +86,7 @@ _PACKAGE_DIRECTORIES = { setuptools.setup( name='grpcio', - version='0.5.0a0', + version='0.5.0a1', ext_modules=[_EXTENSION_MODULE], packages=list(_PACKAGES), package_dir=_PACKAGE_DIRECTORIES,