From 0ff222a23b94a34d03d1cfa4b5c148406e441f4c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 1 Sep 2017 09:41:43 -0700 Subject: [PATCH 1/8] Add pollset_kick stats --- src/core/lib/debug/stats_data.c | 243 +++++++++++++++++- src/core/lib/debug/stats_data.h | 128 +++++++-- src/core/lib/debug/stats_data.yaml | 12 + src/core/lib/iomgr/ev_epoll1_linux.c | 25 +- .../iomgr/ev_epoll_limited_pollers_linux.c | 5 +- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 5 +- src/core/lib/iomgr/ev_epollex_linux.c | 9 +- src/core/lib/iomgr/ev_epollsig_linux.c | 5 +- src/core/lib/iomgr/ev_poll_posix.c | 46 ++-- src/core/lib/iomgr/ev_posix.c | 4 +- src/core/lib/iomgr/ev_posix.h | 2 +- src/core/lib/iomgr/pollset.h | 2 +- .../google_default_credentials.c | 3 +- src/core/lib/surface/call.c | 5 +- src/core/lib/surface/completion_queue.c | 11 +- test/core/http/httpcli_test.c | 2 +- test/core/http/httpscli_test.c | 2 +- test/core/iomgr/endpoint_tests.c | 9 +- test/core/iomgr/fd_posix_test.c | 16 +- test/core/iomgr/resolve_address_test.c | 6 +- test/core/iomgr/tcp_client_posix_test.c | 6 +- test/core/iomgr/tcp_posix_test.c | 11 +- test/core/iomgr/tcp_server_posix_test.c | 4 +- test/core/iomgr/udp_server_test.c | 8 +- test/core/security/oauth2_utils.c | 3 +- .../print_google_default_creds_token.c | 3 +- test/core/security/verify_jwt.c | 3 +- .../surface/concurrent_connectivity_test.c | 3 +- test/core/util/port_server_client.c | 9 +- test/cpp/qps/report.cc | 4 +- 30 files changed, 477 insertions(+), 117 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 9277ee57b23..445dcb67a32 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -24,6 +24,12 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "server_calls_created", "syscall_poll", "syscall_wait", + "pollset_kick", + "pollset_kicked_without_poller", + "pollset_kicked_again", + "pollset_kick_wakeup_fd", + "pollset_kick_wakeup_cv", + "pollset_kick_own_thread", "histogram_slow_lookups", "syscall_write", "syscall_read", @@ -47,10 +53,226 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_queue_drained", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { - "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", - "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "call_initial_size", "poll_events_returned", "tcp_write_size", + "tcp_write_iov_size", "tcp_read_size", "tcp_read_offer", + "tcp_read_iov_size", "http2_send_message_size", }; const double grpc_stats_table_0[64] = {0, + 1, + 2, + 3, + 4, + 5, + 6.03034139457, + 7.27300346702, + 8.77173877401, + 10.5793158863, + 12.759377303, + 15.3886802239, + 18.5597990724, + 22.3843849243, + 26.9970966002, + 32.5603418323, + 39.2699954346, + 47.3622958068, + 57.1221625891, + 68.8932283217, + 83.0899373109, + 100.212137688, + 120.862680427, + 145.768644968, + 175.806938756, + 212.035172047, + 255.72889502, + 308.426508286, + 371.98342802, + 448.637412817, + 541.087352333, + 652.588291771, + 787.066037897, + 949.255381718, + 1144.86680448, + 1380.78753647, + 1665.32404765, + 2008.49450799, + 2422.38151446, + 2921.55750402, + 3523.59783062, + 4249.69957117, + 5125.4278477, + 6181.61594298, + 7455.45090126, + 8991.78283702, + 10844.7040506, + 13079.4535497, + 15774.7140318, + 19025.3822027, + 22945.9099689, + 27674.3341444, + 33377.1365516, + 40255.1056359, + 48550.4059718, + 58555.104571, + 70621.4541917, + 85174.2957114, + 102726.016236, + 123894.589602, + 149425.334448, + 180217.155944, + 217354.195101, + 262144.0}; +const uint8_t grpc_stats_table_1[124] = { + 0, 2, 2, 4, 4, 6, 6, 8, 8, 11, 11, 11, 13, 13, + 15, 15, 17, 17, 20, 20, 20, 21, 23, 23, 26, 26, 26, 28, + 28, 30, 30, 32, 32, 35, 35, 35, 37, 37, 38, 41, 41, 41, + 43, 43, 45, 45, 47, 47, 50, 50, 50, 52, 52, 54, 54, 56, + 56, 58, 58, 60, 60, 62, 62, 65, 65, 65, 67, 67, 69, 69, + 71, 71, 73, 73, 76, 76, 76, 78, 78, 80, 80, 82, 82, 84, + 84, 86, 86, 88, 88, 91, 91, 91, 93, 93, 95, 95, 97, 97, + 100, 100, 100, 101, 103, 103, 106, 106, 106, 108, 108, 110, 110, 112, + 112, 115, 115, 115, 117, 117, 118, 121, 121, 121, 124, 124}; +const double grpc_stats_table_2[128] = {0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29.0367057593, + 30.1117957626, + 31.2266911945, + 32.382865859, + 33.5818481283, + 34.8252229627, + 36.1146340061, + 37.4517857586, + 38.8384458298, + 40.2764472753, + 41.7676910202, + 43.3141483714, + 44.9178636242, + 46.580956764, + 48.3056262695, + 50.0941520181, + 51.9488983005, + 53.8723169454, + 55.8669505614, + 57.9354358972, + 60.0805073281, + 62.3050004699, + 64.6118559278, + 67.0041231836, + 69.4849646266, + 72.0576597342, + 74.7256094075, + 77.4923404661, + 80.3615103113, + 83.3369117602, + 86.4224780597, + 89.6222880862, + 92.940571737, + 96.3817155226, + 99.9502683646, + 103.65094761, + 107.488645265, + 111.468434465, + 115.595576179, + 119.875526164, + 124.313942178, + 128.916691457, + 133.689858475, + 138.639752984, + 143.772918355, + 149.09614023, + 154.616455489, + 160.341161557, + 166.277826044, + 172.434296755, + 178.818712061, + 185.439511656, + 192.305447719, + 199.425596476, + 206.809370205, + 214.466529676, + 222.407197051, + 230.641869269, + 239.181431919, + 248.037173633, + 257.220801006, + 266.744454071, + 276.62072235, + 286.862661493, + 297.48381054, + 308.498209814, + 319.920419488, + 331.765538824, + 344.04922614, + 356.787719506, + 369.997858208, + 383.697105013, + 397.903569249, + 412.636030746, + 427.913964659, + 443.757567219, + 460.187782422, + 477.226329722, + 494.895732741, + 513.219349041, + 532.221401003, + 551.927007848, + 572.36221884, + 593.554047722, + 615.530508428, + 638.320652111, + 661.954605552, + 686.46361098, + 711.880067376, + 738.237573297, + 765.570971297, + 793.91639398, + 823.311311768, + 853.794582433, + 885.406502465, + 918.188860339, + 952.184991756, + 987.439836931, + 1024.0}; +const uint8_t grpc_stats_table_3[166] = { + 0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 11, 12, 14, + 14, 15, 17, 17, 18, 20, 20, 22, 22, 24, 24, 25, 27, 27, + 29, 29, 31, 31, 34, 34, 34, 36, 36, 38, 38, 39, 40, 42, + 42, 43, 44, 46, 46, 47, 49, 49, 50, 52, 52, 54, 54, 55, + 57, 57, 59, 59, 61, 61, 63, 63, 65, 65, 68, 68, 68, 70, + 70, 71, 72, 73, 75, 75, 76, 78, 78, 79, 81, 81, 82, 84, + 84, 86, 86, 87, 89, 89, 91, 91, 93, 93, 95, 95, 97, 97, + 100, 100, 100, 102, 102, 103, 104, 105, 107, 107, 108, 109, 111, 111, + 113, 113, 114, 116, 116, 117, 119, 119, 121, 121, 123, 123, 125, 125, + 127, 127, 129, 129, 131, 131, 134, 134, 134, 135, 136, 137, 139, 139, + 140, 141, 143, 143, 144, 146, 146, 148, 148, 149, 151, 151, 153, 153, + 155, 155, 157, 157, 159, 159, 161, 161, 163, 163, 166, 166}; +const double grpc_stats_table_4[64] = {0, 1, 2, 3, @@ -114,13 +336,13 @@ const double grpc_stats_table_0[64] = {0, 10005134.9318, 12956014.428, 16777216.0}; -const uint8_t grpc_stats_table_1[87] = { +const uint8_t grpc_stats_table_5[87] = { 0, 1, 3, 3, 4, 6, 6, 7, 9, 9, 10, 12, 12, 13, 15, 15, 16, 18, 18, 19, 21, 21, 22, 24, 24, 25, 27, 27, 28, 30, 30, 31, 32, 34, 34, 36, 36, 37, 39, 39, 40, 42, 42, 43, 44, 46, 46, 47, 49, 49, 51, 51, 52, 53, 55, 55, 56, 58, 58, 59, 61, 61, 63, 63, 64, 65, 67, 67, 68, 70, 70, 71, 73, 73, 75, 75, 76, 77, 79, 79, 80, 82, 82, 83, 85, 85, 87}; -const double grpc_stats_table_2[64] = {0, +const double grpc_stats_table_6[64] = {0, 1, 2, 3, @@ -184,12 +406,13 @@ const double grpc_stats_table_2[64] = {0, 860.147148411, 938.504491184, 1024.0}; -const uint8_t grpc_stats_table_3[52] = { +const uint8_t grpc_stats_table_7[52] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52}; -const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320}; -const double *const grpc_stats_histo_bucket_boundaries[6] = { - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0}; +const int grpc_stats_histo_buckets[8] = {64, 128, 64, 64, 64, 64, 64, 64}; +const int grpc_stats_histo_start[8] = {0, 64, 192, 256, 320, 384, 448, 512}; +const double *const grpc_stats_histo_bucket_boundaries[8] = { + grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 4d1078dfdb8..17e5f22e6c2 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -28,6 +28,12 @@ typedef enum { GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, GRPC_STATS_COUNTER_SYSCALL_POLL, GRPC_STATS_COUNTER_SYSCALL_WAIT, + GRPC_STATS_COUNTER_POLLSET_KICK, + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER, + GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV, + GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD, GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, GRPC_STATS_COUNTER_SYSCALL_WRITE, GRPC_STATS_COUNTER_SYSCALL_READ, @@ -53,6 +59,8 @@ typedef enum { } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; typedef enum { + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -63,19 +71,23 @@ typedef enum { } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; typedef enum { - GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_BUCKETS = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_BUCKETS = 128, + GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 192, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 256, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128, + GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 320, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 192, + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 384, GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 256, + GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 448, GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 512, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 384 + GRPC_STATS_HISTOGRAM_BUCKETS = 576 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -85,6 +97,19 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL) #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT) +#define GRPC_STATS_INC_POLLSET_KICK(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK) +#define GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER) +#define GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV) +#define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD) #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS) #define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \ @@ -138,6 +163,55 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED) #define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) +#define GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, value) \ + do { \ + union { \ + double dbl; \ + uint64_t uint; \ + } _val; \ + _val.dbl = (double)(value); \ + if (_val.dbl < 0) _val.dbl = 0; \ + if (_val.dbl < 6.000000) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, (int)_val.dbl); \ + } else { \ + if (_val.uint < 4688247212092686336ull) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_stats_table_1[((_val.uint - 4618441417868443648ull) >> 49)]); \ + } else { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ + grpc_stats_table_0, 64)); \ + } \ + } \ + } while (false) +#define GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, value) \ + do { \ + union { \ + double dbl; \ + uint64_t uint; \ + } _val; \ + _val.dbl = (double)(value); \ + if (_val.dbl < 0) _val.dbl = 0; \ + if (_val.dbl < 29.000000) { \ + GRPC_STATS_INC_HISTOGRAM((exec_ctx), \ + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + (int)_val.dbl); \ + } else { \ + if (_val.uint < 4652218415073722368ull) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + grpc_stats_table_3[((_val.uint - 4628855992006737920ull) >> 47)]); \ + } else { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ + grpc_stats_table_2, 128)); \ + } \ + } \ + } while (false) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ do { \ union { \ @@ -153,12 +227,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -177,12 +251,12 @@ typedef enum { if (_val.uint < 4652218415073722368ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ - grpc_stats_table_3[((_val.uint - 4622945017495814144ull) >> 49)]); \ + grpc_stats_table_7[((_val.uint - 4622945017495814144ull) >> 49)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_2, 64)); \ + grpc_stats_table_6, 64)); \ } \ } \ } while (false) @@ -201,12 +275,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -225,12 +299,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -249,12 +323,12 @@ typedef enum { if (_val.uint < 4652218415073722368ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, \ - grpc_stats_table_3[((_val.uint - 4622945017495814144ull) >> 49)]); \ + grpc_stats_table_7[((_val.uint - 4622945017495814144ull) >> 49)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_2, 64)); \ + grpc_stats_table_6, 64)); \ } \ } \ } while (false) @@ -274,21 +348,25 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) extern const double grpc_stats_table_0[64]; -extern const uint8_t grpc_stats_table_1[87]; -extern const double grpc_stats_table_2[64]; -extern const uint8_t grpc_stats_table_3[52]; -extern const int grpc_stats_histo_buckets[6]; -extern const int grpc_stats_histo_start[6]; -extern const double *const grpc_stats_histo_bucket_boundaries[6]; +extern const uint8_t grpc_stats_table_1[124]; +extern const double grpc_stats_table_2[128]; +extern const uint8_t grpc_stats_table_3[166]; +extern const double grpc_stats_table_4[64]; +extern const uint8_t grpc_stats_table_5[87]; +extern const double grpc_stats_table_6[64]; +extern const uint8_t grpc_stats_table_7[52]; +extern const int grpc_stats_histo_buckets[8]; +extern const int grpc_stats_histo_start[8]; +extern const double *const grpc_stats_histo_bucket_boundaries[8]; #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a0d042a688b..4e0bc5de588 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -18,9 +18,21 @@ # overall - counter: client_calls_created - counter: server_calls_created +- histogram: call_initial_size + max: 262144 + buckets: 64 # polling - counter: syscall_poll - counter: syscall_wait +- histogram: poll_events_returned + max: 1024 + buckets: 128 +- counter: pollset_kick +- counter: pollset_kicked_without_poller +- counter: pollset_kicked_again +- counter: pollset_kick_wakeup_fd +- counter: pollset_kick_wakeup_cv +- counter: pollset_kick_own_thread # stats system - counter: histogram_slow_lookups # tcp diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 7f053fa728b..6a3c2654a8c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -502,12 +502,14 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static grpc_error *pollset_kick_all(grpc_pollset *pollset) { +static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { GPR_TIMER_BEGIN("pollset_kick_all", 0); grpc_error *error = GRPC_ERROR_NONE; if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: break; @@ -550,7 +552,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutdown_closure = closure; pollset->shutting_down = true; - GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(exec_ctx, pollset)); pollset_maybe_finish_shutdown(exec_ctx, pollset); GPR_TIMER_END("pollset_shutdown", 0); } @@ -646,6 +648,8 @@ static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, r); + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "ps: %p poll got %d events", ps, r); } @@ -971,9 +975,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, return error; } -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *ret_err = GRPC_ERROR_NONE; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_strvec log; @@ -1005,6 +1010,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { grpc_pollset_worker *root_worker = pollset->root_worker; if (root_worker == NULL) { + GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx); pollset->kicked_without_poller = true; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked_without_poller"); @@ -1013,12 +1019,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } grpc_pollset_worker *next_worker = root_worker->next; if (root_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; } else if (next_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1029,6 +1037,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, // there is no next worker root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( &g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", root_worker); } @@ -1036,6 +1045,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (next_worker->kick_state == UNKICKED) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1053,10 +1063,12 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } SET_KICK_STATE(root_worker, KICKED); if (root_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&root_worker->cv); } goto done; } else { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker, root_worker); @@ -1066,23 +1078,27 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); GPR_ASSERT(next_worker->kick_state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } goto done; } } else if (specific_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. specific worker already kicked"); } goto done; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker); } @@ -1090,6 +1106,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } else if (specific_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick active poller"); } @@ -1097,6 +1114,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (specific_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick waiting worker"); } @@ -1104,6 +1122,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_cv_signal(&specific_worker->cv); goto done; } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick non-waiting worker"); } diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c index e2e3cd9003f..ee3250297fe 100644 --- a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c @@ -1112,12 +1112,13 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker != NULL) { if (worker == GRPC_POLLSET_KICK_BROADCAST) { if (pollset_has_workers(p)) { @@ -1265,7 +1266,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index ddb8c74d2de..ace491550b0 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -632,9 +632,10 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *error = GRPC_ERROR_NONE; const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; @@ -731,7 +732,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying epoll set. In such a case, we diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0e42f76af37..165a562bd74 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -561,6 +561,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker->pollable != &pollset->pollable) { gpr_mu_lock(&worker->pollable->po.mu); } @@ -599,7 +600,8 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, +static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -662,13 +664,14 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } /* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { pollable *p = pollset->current_pollable; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } - grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + grpc_error *error = pollset_kick_inner(exec_ctx, pollset, p, specific_worker); if (p != &pollset->pollable) { gpr_mu_unlock(&p->po.mu); } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 59c7cdc285f..6cc96455266 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1020,10 +1020,11 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; if (worker != NULL) { @@ -1157,7 +1158,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index fbd265f3ce7..78817144af7 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -209,7 +209,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, #define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 /* As per pollset_kick, with an extended set of flags (defined above) -- mostly for fd_posix's use. */ -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) GRPC_MUST_USE_RESULT; @@ -365,36 +365,39 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, return notifier; } -static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) { +static grpc_error *pollset_kick_locked(grpc_exec_ctx *exec_ctx, + grpc_fd_watcher *watcher) { gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); - grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker, - GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); + grpc_error *err = + pollset_kick_ext(exec_ctx, watcher->pollset, watcher->worker, + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); gpr_mu_unlock(&watcher->pollset->mu); return err; } -static void maybe_wake_one_watcher_locked(grpc_fd *fd) { +static void maybe_wake_one_watcher_locked(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { - pollset_kick_locked(fd->inactive_watcher_root.next); + pollset_kick_locked(exec_ctx, fd->inactive_watcher_root.next); } else if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } else if (fd->write_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } -static void wake_all_watchers_locked(grpc_fd *fd) { +static void wake_all_watchers_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_fd_watcher *watcher; for (watcher = fd->inactive_watcher_root.next; watcher != &fd->inactive_watcher_root; watcher = watcher->next) { - pollset_kick_locked(watcher); + pollset_kick_locked(exec_ctx, watcher); } if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } @@ -435,7 +438,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, if (!has_watchers(fd)) { close_fd_locked(exec_ctx, fd); } else { - wake_all_watchers_locked(fd); + wake_all_watchers_locked(exec_ctx, fd); } gpr_mu_unlock(&fd->mu); UNREF_BY(fd, 2, reason); /* drop the reference */ @@ -479,7 +482,7 @@ static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, /* already ready ==> queue the closure to run immediately */ *st = CLOSURE_NOT_READY; GRPC_CLOSURE_SCHED(exec_ctx, closure, fd_shutdown_error(fd)); - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } else { /* upcallptr was set to a different closure. This is an error! */ gpr_log(GPR_ERROR, @@ -648,7 +651,7 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, } } if (kick) { - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { close_fd_locked(exec_ctx, fd); @@ -712,11 +715,12 @@ static void kick_append_error(grpc_error **composite, grpc_error *error) { *composite = grpc_error_add_child(*composite, error); } -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { GPR_TIMER_BEGIN("pollset_kick_ext", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); /* pollset->mu already held */ if (specific_worker != NULL) { @@ -782,9 +786,9 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p, return error; } -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { - return pollset_kick_ext(p, specific_worker, 0); + return pollset_kick_ext(exec_ctx, p, specific_worker, 0); } /* global state management */ @@ -847,7 +851,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } pollset->fds[pollset->fd_count++] = fd; GRPC_FD_REF(fd, "multipoller"); - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); exit: gpr_mu_unlock(&pollset->mu); } @@ -1070,7 +1074,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* check shutdown conditions */ if (pollset->shutting_down) { if (pollset_has_workers(pollset)) { - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); } else if (!pollset->called_shutdown && !pollset_has_observers(pollset)) { pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); @@ -1099,7 +1103,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset_has_workers(pollset)) { GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pollset->idle_jobs); } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 424b40e425b..32363765e39 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -214,9 +214,9 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline); } -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - return g_event_engine->pollset_kick(pollset, specific_worker); + return g_event_engine->pollset_kick(exec_ctx, pollset, specific_worker); } void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1108e46ef81..1ff2ff14133 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -54,7 +54,7 @@ typedef struct grpc_event_engine_vtable { grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); - grpc_error *(*pollset_kick)(grpc_pollset *pollset, + grpc_error *(*pollset_kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); void (*pollset_add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index a609a3877a7..a0f6b3a9d3a 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -76,7 +76,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* Break one polling thread out of polling work for this pollset. If specific_worker is non-NULL, then kick that worker. */ -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index a2a8e289ee3..691d66df698 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -79,7 +79,8 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, detector->is_done = 1; GRPC_LOG_IF_ERROR( "Pollset kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL)); + grpc_pollset_kick(exec_ctx, + grpc_polling_entity_pollset(&detector->pollent), NULL)); gpr_mu_unlock(g_polling_mu); } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index e68c201134d..2599765510f 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -322,8 +322,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, grpc_channel_get_channel_stack(args->channel); grpc_call *call; GPR_TIMER_BEGIN("grpc_call_create", 0); - gpr_arena *arena = - gpr_arena_create(grpc_channel_get_call_size_estimate(args->channel)); + size_t initial_size = grpc_channel_get_call_size_estimate(args->channel); + GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, initial_size); + gpr_arena *arena = gpr_arena_create(initial_size); call = gpr_arena_alloc(arena, sizeof(grpc_call) + channel_stack->call_stack_size); gpr_ref_init(&call->ext_ref, 1); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 10e4e5ab0cb..a15b09e55d1 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -54,7 +54,7 @@ typedef struct { bool can_listen; size_t (*size)(void); void (*init)(grpc_pollset *pollset, gpr_mu **mu); - grpc_error *(*kick)(grpc_pollset *pollset, + grpc_error *(*kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); grpc_error *(*work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, @@ -130,7 +130,8 @@ static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, } static grpc_error *non_polling_poller_kick( - grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { non_polling_poller *p = (non_polling_poller *)pollset; if (specific_worker == NULL) specific_worker = (grpc_pollset_worker *)p->root; if (specific_worker != NULL) { @@ -645,7 +646,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, if (is_first) { gpr_mu_lock(cq->mu); grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); if (kick_error != GRPC_ERROR_NONE) { @@ -731,7 +732,7 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, } grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), pluck_worker); gpr_mu_unlock(cq->mu); @@ -930,7 +931,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, if (cq_event_queue_num_items(&cqd->queue) > 0 && gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(&exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); } diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index b8b96d673ce..8a53903763d 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index a9d7abdcffd..c7455bd8df3 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 11b45e8e084..353d79b11d1 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -125,7 +125,8 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Read handler done"); gpr_mu_lock(g_mu); state->read_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } else if (error == GRPC_ERROR_NONE) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, @@ -160,7 +161,8 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Write handler done"); gpr_mu_lock(g_mu); state->write_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } @@ -252,7 +254,8 @@ static void inc_on_failure(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_mu_lock(g_mu); *(int *)arg += (error != GRPC_ERROR_NONE); - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT( + GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 5791d17af62..881277a8d6a 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -178,8 +178,8 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, gpr_mu_lock(g_mu); sv->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -297,8 +297,8 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, false /* already_closed */, "c"); cl->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Write as much as possible, then register notify_on_write. */ @@ -417,8 +417,8 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = first_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -429,8 +429,8 @@ static void second_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = second_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index 7f5c4073d81..1110c04b6e2 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -106,7 +106,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *argsp, GPR_ASSERT(args->addrs->naddrs > 0); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } @@ -115,7 +116,8 @@ static void must_fail(grpc_exec_ctx *exec_ctx, void *argsp, grpc_error *err) { GPR_ASSERT(err != GRPC_ERROR_NONE); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 00ea495bbe3..1032da942b3 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -53,8 +53,10 @@ static gpr_timespec test_deadline(void) { static void finish_connection() { gpr_mu_lock(g_mu); g_connections_complete++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(&exec_ctx, g_pollset, NULL))); + grpc_exec_ctx_finish(&exec_ctx); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index c45068e7ec5..8cfc1bcab1a 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -147,7 +147,8 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, gpr_log(GPR_INFO, "Read %" PRIuPTR " bytes of %" PRIuPTR, read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); @@ -295,8 +296,8 @@ static void write_done(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -406,8 +407,8 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *errors) { int *done = arg; *done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Do a read_test, then release fd and try to read/write again. Verify that diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 2371721a608..4d84608376a 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -159,8 +159,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_lock(g_mu); g_result = temp_result; g_nconnects++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index aa34857dbd1..1d051bea62d 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -61,8 +61,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { g_number_of_reads++; g_number_of_bytes_read += (int)byte_count; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -70,8 +70,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { gpr_mu_lock(g_mu); g_number_of_writes++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index fdbc6ea7417..d240403a293 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -60,7 +60,8 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *arg, request->token = token; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&request->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&request->pops), + NULL)); gpr_mu_unlock(request->mu); } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index e1385a80caa..3144717a853 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -57,7 +57,8 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *arg, sync->is_done = true; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&sync->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&sync->pops), + NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 259038f4d6c..5faa6352a87 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -66,7 +66,8 @@ static void on_jwt_verification_done(grpc_exec_ctx *exec_ctx, void *user_data, gpr_mu_lock(sync->mu); sync->is_done = 1; - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(sync->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, sync->pollset, NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 08079b6091a..ec2cd8610bc 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -108,7 +108,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *vargs, grpc_endpoint *tcp, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Connected")); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index d5739effb32..1b015c95e98 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -54,7 +54,8 @@ static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->done = 1; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } @@ -153,7 +154,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = 0; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); return; } @@ -189,7 +191,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = port; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 3c99bda1447..42ebeff41d9 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -107,8 +107,8 @@ void GprLogReporter::ReportCoreStats(const char* name, int idx, grpc_stats_counter_name[i], data.counters[i]); } for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) { - gpr_log(GPR_DEBUG, "%s[%d].%s = %lf/%lf/%lf (50/95/99%%-ile)", name, idx, - grpc_stats_histogram_name[i], + gpr_log(GPR_DEBUG, "%s[%d].%s = %.1lf/%.1lf/%.1lf (50/95/99%%-ile)", name, + idx, grpc_stats_histogram_name[i], grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 50), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 95), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 99)); From 8e87c74fe95e445dac4dc66d4f464d572fcbaa5e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 14:06:09 -0700 Subject: [PATCH 2/8] Fix compilation for C++ --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 9d71d3990d4..5c9405f583d 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -59,7 +59,8 @@ static void pollset_destroy(grpc_exec_ctx* exec_ctx, grpc_pollset* ps) { gpr_mu_destroy(&ps->mu); } -static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) { +static grpc_error* pollset_kick(grpc_exec_ctx* exec_ctx, grpc_pollset* p, + grpc_pollset_worker* worker) { return GRPC_ERROR_NONE; } From 480f5d861264f00ee2226b2aa000bbeb268f0a29 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:36:07 -0700 Subject: [PATCH 3/8] Add missing increments --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 322fa160d08..8a984e24eb8 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -512,14 +512,17 @@ static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { gpr_cv_signal(&worker->cv); } break; case DESIGNATED_POLLER: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); SET_KICK_STATE(worker, KICKED); append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), "pollset_kick_all"); From ebacb2f20d5b9fbf73d1f90d06eee7553c93aee6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 12:32:33 -0700 Subject: [PATCH 4/8] Fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index bf080fff0c0..f7f6cec9ef6 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -515,9 +515,9 @@ static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: - GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&worker->cv); } break; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index f1a3b97f4da..4e5bed5016b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -666,12 +666,8 @@ static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { -<<<<<<< HEAD - pollable *p = pollset->current_pollable; - GRPC_STATS_INC_POLLSET_KICK(exec_ctx); -======= pollable_t *p = pollset->current_pollable; ->>>>>>> d1fefe89dacf198a721053f2b1a3e173f1f24948 + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } From cf34fa58f476aae77c4e2dca0ff866542dd3af28 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 12:37:01 -0700 Subject: [PATCH 5/8] Document only epoll1 for these stats, add missing spot --- src/core/lib/debug/stats_data.c | 14 ++++++++------ src/core/lib/debug/stats_data.yaml | 6 ++++++ src/core/lib/iomgr/ev_epoll1_linux.c | 8 +++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index fc1ac223de3..1a020a15c38 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -70,16 +70,18 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of server side calls created by this process", "Number of polling syscalls (epoll_wait, poll, etc) made by this process", "Number of sleeping syscalls made by this process", - "How many polling wakeups were performed by the process", - "How many times was a polling wakeup requested without an active poller", + "How many polling wakeups were performed by the process (only valid for " + "epoll1 right now)", + "How many times was a polling wakeup requested without an active poller " + "(only valid for epoll1 right now)", "How many times was the same polling worker awoken repeatedly before " - "waking up", + "waking up (only valid for epoll1 right now)", "How many times was an eventfd used as the wakeup vector for a polling " - "wakeup", + "wakeup (only valid for epoll1 right now)", "How many times was a condition variable used as the wakeup vector for a " - "polling wakeup", + "polling wakeup (only valid for epoll1 right now)", "How many times could a polling wakeup be satisfied by keeping the waking " - "thread awake?", + "thread awake? (only valid for epoll1 right now)", "Number of times histogram increments went through the slow (binary " "search) path", "Number of write syscalls (or equivalent - eg sendmsg) made by this " diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 26c8e1b11d2..b43c466d4f0 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -35,20 +35,26 @@ doc: How many events are called for each syscall_poll - counter: pollset_kick doc: How many polling wakeups were performed by the process + (only valid for epoll1 right now) - counter: pollset_kicked_without_poller doc: How many times was a polling wakeup requested without an active poller + (only valid for epoll1 right now) - counter: pollset_kicked_again doc: How many times was the same polling worker awoken repeatedly before waking up + (only valid for epoll1 right now) - counter: pollset_kick_wakeup_fd doc: How many times was an eventfd used as the wakeup vector for a polling wakeup + (only valid for epoll1 right now) - counter: pollset_kick_wakeup_cv doc: How many times was a condition variable used as the wakeup vector for a polling wakeup + (only valid for epoll1 right now) - counter: pollset_kick_own_thread doc: How many times could a polling wakeup be satisfied by keeping the waking thread awake? + (only valid for epoll1 right now) # stats system - counter: histogram_slow_lookups doc: Number of times histogram increments went through the slow diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index f7f6cec9ef6..8f661583696 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -789,7 +789,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood) { + grpc_exec_ctx *exec_ctx, pollset_neighbourhood *neighbourhood) { GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0); bool found_worker = false; do { @@ -813,6 +813,7 @@ static bool check_neighbourhood_for_available_poller( SET_KICK_STATE(inspect_worker, DESIGNATED_POLLER); if (inspect_worker->initialized_cv) { GPR_TIMER_MARK("signal worker", 0); + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&inspect_worker->cv); } } else { @@ -891,7 +892,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; if (gpr_mu_trylock(&neighbourhood->mu)) { found_worker = - check_neighbourhood_for_available_poller(neighbourhood); + check_neighbourhood_for_available_poller(exec_ctx, neighbourhood); gpr_mu_unlock(&neighbourhood->mu); scan_state[i] = true; } else { @@ -904,7 +905,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, &g_neighbourhoods[(poller_neighbourhood_idx + i) % g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); - found_worker = check_neighbourhood_for_available_poller(neighbourhood); + found_worker = + check_neighbourhood_for_available_poller(exec_ctx, neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); From 1a012bbe8fceacfff9a2fc49528ad07de75743a2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 14:29:00 -0700 Subject: [PATCH 6/8] Review feedback --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8f855a7b6ec..aa861d3cc19 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -873,6 +873,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); SET_KICK_STATE(worker->next, DESIGNATED_POLLER); + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { gpr_mu_unlock(&pollset->mu); @@ -1098,7 +1099,7 @@ static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, goto done; } } else { - GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } From 7f10c299c7eb5d6688c20ae43c63b0548ca8acd1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:26:56 -0700 Subject: [PATCH 7/8] Windows --- src/core/lib/iomgr/pollset_windows.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index ea017a60546..eb295d3eebd 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -98,7 +98,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { pollset->shutting_down = 1; - grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + grpc_pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_NONE); } else { @@ -181,7 +181,7 @@ done: return GRPC_ERROR_NONE; } -grpc_error *grpc_pollset_kick(grpc_pollset *p, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker != NULL) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { @@ -209,7 +209,7 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p, specific_worker = pop_front_worker(&p->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET); if (specific_worker != NULL) { - grpc_pollset_kick(p, specific_worker); + grpc_pollset_kick(exec_ctx, p, specific_worker); } else if (p->is_iocp_worker) { grpc_iocp_kick(); } else { From 350a4239fe3a56fb20b3eea99badaf307c652ad5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 18 Sep 2017 22:41:24 -0700 Subject: [PATCH 8/8] Fix compile error --- test/cpp/naming/resolver_component_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 0857fb6a32c..396f4ceb9e1 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -270,7 +270,8 @@ void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, CheckLBPolicyResultLocked(channel_args, args); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); }