diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 72f418b3d7a..c4783d9b4d2 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -59,7 +59,7 @@ #include "src/core/ext/transport/chttp2/transport/stream_map.h" #include "src/core/ext/transport/chttp2/transport/varint.h" #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/debug/stats_data.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/bitset.h" diff --git a/src/core/lib/debug/stats.cc b/src/core/lib/debug/stats.cc index 307abb4e49d..606839335e4 100644 --- a/src/core/lib/debug/stats.cc +++ b/src/core/lib/debug/stats.cc @@ -67,21 +67,12 @@ void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a, } } -int grpc_stats_histo_find_bucket_slow(int value, const int* table, - int table_size) { - GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(); - const int* const start = table; - while (table_size > 0) { - int step = table_size / 2; - const int* it = table + step; - if (value >= *it) { - table = it + 1; - table_size -= step + 1; - } else { - table_size = step; - } - } - return static_cast(table - start) - 1; +void grpc_stats_inc_histogram_value(int histogram, int value) { + const int bucket = grpc_stats_get_bucket[histogram](value); + gpr_atm_no_barrier_fetch_add( + &GRPC_THREAD_STATS_DATA() + ->histograms[grpc_stats_histo_start[histogram] + bucket], + 1); } size_t grpc_stats_histo_count(const grpc_stats_data* stats, diff --git a/src/core/lib/debug/stats.h b/src/core/lib/debug/stats.h index 2ecfec50842..be19197e211 100644 --- a/src/core/lib/debug/stats.h +++ b/src/core/lib/debug/stats.h @@ -27,7 +27,7 @@ #include -#include "src/core/lib/debug/stats_data.h" +#include "src/core/lib/debug/stats_data.h" // IWYU pragma: export #include "src/core/lib/iomgr/exec_ctx.h" typedef struct grpc_stats_data { @@ -60,12 +60,11 @@ void grpc_stats_collect(grpc_stats_data* output); void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a, grpc_stats_data* c); std::string grpc_stats_data_as_json(const grpc_stats_data* data); -int grpc_stats_histo_find_bucket_slow(int value, const int* table, - int table_size); double grpc_stats_histo_percentile(const grpc_stats_data* stats, grpc_stats_histograms histogram, double percentile); size_t grpc_stats_histo_count(const grpc_stats_data* stats, grpc_stats_histograms histogram); +void grpc_stats_inc_histogram_value(int histogram, int value); #endif // GRPC_CORE_LIB_DEBUG_STATS_H diff --git a/src/core/lib/debug/stats_data.cc b/src/core/lib/debug/stats_data.cc index 6b46a1ff1cf..b9111fed8bf 100644 --- a/src/core/lib/debug/stats_data.cc +++ b/src/core/lib/debug/stats_data.cc @@ -22,21 +22,34 @@ #include "src/core/lib/debug/stats_data.h" -#include +#include #include "src/core/lib/debug/stats.h" -#include "src/core/lib/gpr/useful.h" +namespace { +union DblUint { + double dbl; + uint64_t uint; +}; +} // namespace const char* grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { - "client_calls_created", "server_calls_created", - "client_channels_created", "client_subchannels_created", - "server_channels_created", "histogram_slow_lookups", - "syscall_write", "syscall_read", - "tcp_read_alloc_8k", "tcp_read_alloc_64k", - "http2_settings_writes", "http2_pings_sent", - "http2_writes_begun", "http2_transport_stalls", - "http2_stream_stalls", "cq_pluck_creates", - "cq_next_creates", "cq_callback_creates", + "client_calls_created", + "server_calls_created", + "client_channels_created", + "client_subchannels_created", + "server_channels_created", + "syscall_write", + "syscall_read", + "tcp_read_alloc_8k", + "tcp_read_alloc_64k", + "http2_settings_writes", + "http2_pings_sent", + "http2_writes_begun", + "http2_transport_stalls", + "http2_stream_stalls", + "cq_pluck_creates", + "cq_next_creates", + "cq_callback_creates", }; const char* grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of client side calls created by this process", @@ -44,8 +57,6 @@ const char* grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of client channels created", "Number of client subchannels created", "Number of server channels created", - "Number of times histogram increments went through the slow (binary " - "search) path", "Number of write syscalls (or equivalent - eg sendmsg) made by this " "process", "Number of read syscalls (or equivalent - eg recvmsg) made by this process", @@ -80,22 +91,6 @@ const char* grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "Size of messages received by HTTP2 transport", }; const int grpc_stats_table_0[65] = { - 0, 1, 2, 3, 4, 5, 7, 9, 11, 14, - 17, 21, 26, 32, 39, 47, 57, 68, 82, 98, - 117, 140, 167, 199, 238, 284, 339, 404, 482, 575, - 685, 816, 972, 1158, 1380, 1644, 1959, 2334, 2780, 3312, - 3945, 4699, 5597, 6667, 7941, 9459, 11267, 13420, 15984, 19038, - 22676, 27009, 32169, 38315, 45635, 54353, 64737, 77104, 91834, 109378, - 130273, 155159, 184799, 220100, 262144}; -const uint8_t grpc_stats_table_1[124] = { - 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, - 7, 7, 7, 8, 9, 9, 10, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, - 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 22, 23, 24, - 24, 25, 25, 26, 26, 26, 27, 27, 28, 29, 29, 30, 30, 30, 31, 31, 32, 33, - 33, 34, 34, 34, 35, 35, 36, 37, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41, - 42, 42, 43, 43, 44, 44, 45, 45, 46, 46, 47, 47, 48, 48, 49, 49, 50, 50, - 51, 51, 52, 52, 53, 53, 54, 54, 55, 55, 56, 56, 57, 57, 58, 58}; -const int grpc_stats_table_2[65] = { 0, 1, 2, 3, 4, 6, 8, 11, 15, 20, 26, 34, 44, 57, 73, 94, 121, 155, 199, 255, 327, 419, 537, 688, @@ -105,201 +100,117 @@ const int grpc_stats_table_2[65] = { 326126, 417200, 533707, 682750, 873414, 1117323, 1429345, 1828502, 2339127, 2992348, 3827987, 4896985, 6264509, 8013925, 10251880, 13114801, 16777216}; -const uint8_t grpc_stats_table_3[87] = { - 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, - 11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23, - 24, 25, 25, 26, 27, 27, 28, 29, 29, 30, 31, 31, 32, 33, 34, 34, 35, 36, - 36, 37, 38, 39, 39, 40, 41, 41, 42, 43, 44, 44, 45, 45, 46, 47, 48, 48, - 49, 50, 51, 51, 52, 53, 53, 54, 55, 56, 56, 57, 58, 58, 59}; -const int grpc_stats_table_4[65] = { +const uint8_t grpc_stats_table_1[87] = { + 5, 5, 6, 6, 7, 8, 8, 9, 9, 10, 11, 11, 12, 13, 13, 14, 15, 16, + 16, 17, 18, 18, 19, 20, 20, 21, 22, 22, 23, 24, 25, 25, 26, 27, 27, 28, + 29, 30, 30, 31, 32, 32, 33, 34, 34, 35, 36, 36, 37, 38, 39, 39, 40, 41, + 41, 42, 43, 44, 44, 45, 46, 46, 47, 48, 49, 49, 50, 50, 51, 52, 53, 53, + 54, 55, 56, 56, 57, 58, 58, 59, 60, 61, 61, 62, 63, 63, 64}; +const int grpc_stats_table_2[65] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30, 33, 36, 39, 43, 47, 51, 56, 61, 66, 72, 78, 85, 92, 100, 109, 118, 128, 139, 151, 164, 178, 193, 209, 226, 244, 264, 285, 308, 333, 359, 387, 418, 451, 486, 524, 565, 609, 656, 707, 762, 821, 884, 952, 1024}; -const uint8_t grpc_stats_table_5[102] = { - 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, - 6, 7, 7, 7, 8, 8, 9, 9, 10, 11, 11, 12, 12, 13, 13, 14, 14, - 14, 15, 15, 16, 16, 17, 17, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, - 23, 24, 24, 24, 25, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, - 32, 33, 33, 34, 35, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41, - 42, 42, 43, 44, 44, 45, 46, 46, 47, 48, 48, 49, 49, 50, 50, 51, 51}; -void grpc_stats_inc_call_initial_size(int value) { - value = grpc_core::Clamp(value, 0, 262144); - if (value < 6) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4651092515166879744ull) { - int bucket = - grpc_stats_table_1[((_val.uint - 4618441417868443648ull) >> 49)] + 6; - _bkt.dbl = grpc_stats_table_0[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, bucket); - return; - } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_0, 64)); -} -void grpc_stats_inc_tcp_write_size(int value) { - value = grpc_core::Clamp(value, 0, 16777216); - if (value < 5) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4683743612465315840ull) { - int bucket = - grpc_stats_table_3[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_2[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, bucket); - return; - } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_2, 64)); -} -void grpc_stats_inc_tcp_write_iov_size(int value) { - value = grpc_core::Clamp(value, 0, 1024); - if (value < 13) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4637863191261478912ull) { - int bucket = - grpc_stats_table_5[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_4[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, bucket); - return; - } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_4, 64)); -} -void grpc_stats_inc_tcp_read_size(int value) { - value = grpc_core::Clamp(value, 0, 16777216); - if (value < 5) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4683743612465315840ull) { - int bucket = - grpc_stats_table_3[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_2[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, bucket); - return; - } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_2, 64)); -} -void grpc_stats_inc_tcp_read_offer(int value) { - value = grpc_core::Clamp(value, 0, 16777216); +const uint8_t grpc_stats_table_3[102] = { + 13, 13, 13, 14, 14, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, + 19, 20, 20, 20, 21, 21, 22, 22, 23, 24, 24, 25, 25, 26, 26, 27, 27, + 27, 28, 28, 29, 29, 30, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, + 36, 37, 37, 37, 38, 39, 40, 40, 41, 41, 42, 42, 43, 43, 44, 44, 45, + 45, 46, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 53, 53, 54, 54, + 55, 55, 56, 57, 57, 58, 59, 59, 60, 61, 61, 62, 62, 63, 63, 64, 64}; +const int grpc_stats_table_4[65] = { + 0, 1, 2, 3, 4, 5, 7, 9, 11, 14, + 17, 21, 26, 32, 39, 47, 57, 68, 82, 98, + 117, 140, 167, 199, 238, 284, 339, 404, 482, 575, + 685, 816, 972, 1158, 1380, 1644, 1959, 2334, 2780, 3312, + 3945, 4699, 5597, 6667, 7941, 9459, 11267, 13420, 15984, 19038, + 22676, 27009, 32169, 38315, 45635, 54353, 64737, 77104, 91834, 109378, + 130273, 155159, 184799, 220100, 262144}; +const uint8_t grpc_stats_table_5[124] = { + 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 12, + 13, 13, 13, 14, 15, 15, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, + 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 28, 29, 30, + 30, 31, 31, 32, 32, 32, 33, 33, 34, 35, 35, 36, 36, 36, 37, 37, 38, 39, + 39, 40, 40, 40, 41, 41, 42, 43, 43, 43, 44, 44, 45, 45, 46, 46, 47, 47, + 48, 48, 49, 49, 50, 50, 51, 51, 52, 52, 53, 53, 54, 54, 55, 55, 56, 56, + 57, 57, 58, 58, 59, 59, 60, 60, 61, 61, 62, 62, 63, 63, 64, 64}; +namespace grpc_core { +int BucketForHistogramValue_16777216_64(int value) { if (value < 5) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, value); - return; + if (value < 0) { + return 0; + } else { + return value; + } + } else { + if (value < 14680065) { + // first_nontrivial_code=4617315517961601024 + // last_code=4714142909950066688 [14680064.000000] + DblUint val; + val.dbl = value; + const int bucket = + grpc_stats_table_1[((val.uint - 4617315517961601024ull) >> 50)]; + return bucket - (value < grpc_stats_table_0[bucket]); + } else { + return 63; + } } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4683743612465315840ull) { - int bucket = - grpc_stats_table_3[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_2[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, bucket); - return; - } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_2, 64)); } -void grpc_stats_inc_tcp_read_offer_iov_size(int value) { - value = grpc_core::Clamp(value, 0, 1024); +int BucketForHistogramValue_1024_64(int value) { if (value < 13) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE, - value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4637863191261478912ull) { - int bucket = - grpc_stats_table_5[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_4[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE, - bucket); - return; + if (value < 0) { + return 0; + } else { + return value; + } + } else { + if (value < 993) { + // first_nontrivial_code=4623507967449235456 + // last_code=4651936940097011712 [992.000000] + DblUint val; + val.dbl = value; + const int bucket = + grpc_stats_table_3[((val.uint - 4623507967449235456ull) >> 48)]; + return bucket - (value < grpc_stats_table_2[bucket]); + } else { + return 63; + } } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_4, 64)); } -void grpc_stats_inc_http2_send_message_size(int value) { - value = grpc_core::Clamp(value, 0, 16777216); - if (value < 5) { - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, - value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4683743612465315840ull) { - int bucket = - grpc_stats_table_3[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_2[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, - bucket); - return; +int BucketForHistogramValue_262144_64(int value) { + if (value < 6) { + if (value < 0) { + return 0; + } else { + return value; + } + } else { + if (value < 245761) { + // first_nontrivial_code=4618441417868443648 + // last_code=4687684262139265024 [245760.000000] + DblUint val; + val.dbl = value; + const int bucket = + grpc_stats_table_5[((val.uint - 4618441417868443648ull) >> 49)]; + return bucket - (value < grpc_stats_table_4[bucket]); + } else { + return 63; + } } - GRPC_STATS_INC_HISTOGRAM( - GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, - grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_2, 64)); } +} // namespace grpc_core const int grpc_stats_histo_buckets[7] = {64, 64, 64, 64, 64, 64, 64}; const int grpc_stats_histo_start[7] = {0, 64, 128, 192, 256, 320, 384}; const int* const grpc_stats_histo_bucket_boundaries[7] = { - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4, - grpc_stats_table_2, grpc_stats_table_2, grpc_stats_table_4, - grpc_stats_table_2}; -void (*const grpc_stats_inc_histogram[7])(int x) = { - grpc_stats_inc_call_initial_size, - grpc_stats_inc_tcp_write_size, - grpc_stats_inc_tcp_write_iov_size, - grpc_stats_inc_tcp_read_size, - grpc_stats_inc_tcp_read_offer, - grpc_stats_inc_tcp_read_offer_iov_size, - grpc_stats_inc_http2_send_message_size}; + grpc_stats_table_4, grpc_stats_table_0, grpc_stats_table_2, + grpc_stats_table_0, grpc_stats_table_0, grpc_stats_table_2, + grpc_stats_table_0}; +int (*const grpc_stats_get_bucket[7])(int value) = { + grpc_core::BucketForHistogramValue_262144_64, + grpc_core::BucketForHistogramValue_16777216_64, + grpc_core::BucketForHistogramValue_1024_64, + grpc_core::BucketForHistogramValue_16777216_64, + grpc_core::BucketForHistogramValue_16777216_64, + grpc_core::BucketForHistogramValue_1024_64, + grpc_core::BucketForHistogramValue_16777216_64}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index f47af826147..24d8a706385 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -23,13 +23,14 @@ #include +// IWYU pragma: private, include "src/core/lib/debug/stats.h" + typedef enum { GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED, GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, GRPC_STATS_COUNTER_CLIENT_CHANNELS_CREATED, GRPC_STATS_COUNTER_CLIENT_SUBCHANNELS_CREATED, GRPC_STATS_COUNTER_SERVER_CHANNELS_CREATED, - GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, GRPC_STATS_COUNTER_SYSCALL_WRITE, GRPC_STATS_COUNTER_SYSCALL_READ, GRPC_STATS_COUNTER_TCP_READ_ALLOC_8K, @@ -85,8 +86,6 @@ typedef enum { GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_CLIENT_SUBCHANNELS_CREATED) #define GRPC_STATS_INC_SERVER_CHANNELS_CREATED() \ GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_SERVER_CHANNELS_CREATED) -#define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS() \ - GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS) #define GRPC_STATS_INC_SYSCALL_WRITE() \ GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_SYSCALL_WRITE) #define GRPC_STATS_INC_SYSCALL_READ() \ @@ -112,29 +111,41 @@ typedef enum { #define GRPC_STATS_INC_CQ_CALLBACK_CREATES() \ GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_CQ_CALLBACK_CREATES) #define GRPC_STATS_INC_CALL_INITIAL_SIZE(value) \ - grpc_stats_inc_call_initial_size((int)(value)) -void grpc_stats_inc_call_initial_size(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_core::BucketForHistogramValue_262144_64(static_cast(value))) #define GRPC_STATS_INC_TCP_WRITE_SIZE(value) \ - grpc_stats_inc_tcp_write_size((int)(value)) -void grpc_stats_inc_tcp_write_size(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ + grpc_core::BucketForHistogramValue_16777216_64(static_cast(value))) #define GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(value) \ - grpc_stats_inc_tcp_write_iov_size((int)(value)) -void grpc_stats_inc_tcp_write_iov_size(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ + grpc_core::BucketForHistogramValue_1024_64(static_cast(value))) #define GRPC_STATS_INC_TCP_READ_SIZE(value) \ - grpc_stats_inc_tcp_read_size((int)(value)) -void grpc_stats_inc_tcp_read_size(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ + grpc_core::BucketForHistogramValue_16777216_64(static_cast(value))) #define GRPC_STATS_INC_TCP_READ_OFFER(value) \ - grpc_stats_inc_tcp_read_offer((int)(value)) -void grpc_stats_inc_tcp_read_offer(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ + grpc_core::BucketForHistogramValue_16777216_64(static_cast(value))) #define GRPC_STATS_INC_TCP_READ_OFFER_IOV_SIZE(value) \ - grpc_stats_inc_tcp_read_offer_iov_size((int)(value)) -void grpc_stats_inc_tcp_read_offer_iov_size(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE, \ + grpc_core::BucketForHistogramValue_1024_64(static_cast(value))) #define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(value) \ - grpc_stats_inc_http2_send_message_size((int)(value)) -void grpc_stats_inc_http2_send_message_size(int x); + GRPC_STATS_INC_HISTOGRAM( \ + GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ + grpc_core::BucketForHistogramValue_16777216_64(static_cast(value))) +namespace grpc_core { +int BucketForHistogramValue_16777216_64(int value); +int BucketForHistogramValue_1024_64(int value); +int BucketForHistogramValue_262144_64(int value); +} // namespace grpc_core extern const int grpc_stats_histo_buckets[7]; extern const int grpc_stats_histo_start[7]; extern const int* const grpc_stats_histo_bucket_boundaries[7]; -extern void (*const grpc_stats_inc_histogram[7])(int x); +extern int (*const grpc_stats_get_bucket[7])(int value); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index fc3ad9ca555..c8c312a8dc5 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -30,10 +30,6 @@ doc: Number of client subchannels created - counter: server_channels_created doc: Number of server channels created -# stats system -- counter: histogram_slow_lookups - doc: Number of times histogram increments went through the slow - (binary search) path # tcp - counter: syscall_write doc: Number of write syscalls (or equivalent - eg sendmsg) made by this process diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 5f4637138d6..f7300f4e04a 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -3,7 +3,6 @@ server_calls_created_per_iteration:FLOAT, client_channels_created_per_iteration:FLOAT, client_subchannels_created_per_iteration:FLOAT, server_channels_created_per_iteration:FLOAT, -histogram_slow_lookups_per_iteration:FLOAT, syscall_write_per_iteration:FLOAT, syscall_read_per_iteration:FLOAT, tcp_read_alloc_8k_per_iteration:FLOAT, diff --git a/src/cpp/util/core_stats.cc b/src/cpp/util/core_stats.cc index de0f2188ee0..a0380663c24 100644 --- a/src/cpp/util/core_stats.cc +++ b/src/cpp/util/core_stats.cc @@ -25,8 +25,6 @@ #include #include -#include "src/core/lib/debug/stats_data.h" - // IWYU pragma: no_include using grpc::core::Bucket; diff --git a/test/core/debug/stats_test.cc b/test/core/debug/stats_test.cc index 5d1d21e8bd8..c168f605cc7 100644 --- a/test/core/debug/stats_test.cc +++ b/test/core/debug/stats_test.cc @@ -87,18 +87,35 @@ static int FindExpectedBucket(int i, int j) { class HistogramTest : public ::testing::TestWithParam {}; +TEST_P(HistogramTest, CheckBucket) { + const int kHistogram = GetParam(); + int max_bucket_boundary = + grpc_stats_histo_bucket_boundaries[kHistogram] + [grpc_stats_histo_buckets[kHistogram] - + 1]; + for (int i = -1000; i < max_bucket_boundary + 1000; i++) { + ASSERT_EQ(FindExpectedBucket(kHistogram, i), + grpc_stats_get_bucket[kHistogram](i)) + << "i=" << i << " expect_bucket=" + << grpc_stats_histo_bucket_boundaries[kHistogram] + [FindExpectedBucket(kHistogram, i)] + << " actual_bucket=" + << grpc_stats_histo_bucket_boundaries[kHistogram] + [grpc_stats_get_bucket[kHistogram]( + i)]; + } +} + TEST_P(HistogramTest, IncHistogram) { const int kHistogram = GetParam(); std::queue threads; auto run = [kHistogram](const std::vector& test_values, int expected_bucket) { - gpr_log(GPR_DEBUG, "expected_bucket:%d nvalues=%" PRIdPTR, expected_bucket, - test_values.size()); grpc_core::ExecCtx exec_ctx; for (auto j : test_values) { std::unique_ptr snapshot(new Snapshot); - grpc_stats_inc_histogram[kHistogram](j); + grpc_stats_inc_histogram_value(kHistogram, j); auto delta = snapshot->delta(); diff --git a/test/core/end2end/tests/simple_request.cc b/test/core/end2end/tests/simple_request.cc index 9bb0b3e8b28..8b328ccb628 100644 --- a/test/core/end2end/tests/simple_request.cc +++ b/test/core/end2end/tests/simple_request.cc @@ -30,7 +30,6 @@ #include #include "src/core/lib/debug/stats.h" -#include "src/core/lib/debug/stats_data.h" #include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/end2end_tests.h" #include "test/core/util/test_config.h" diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 39d13a7f701..2281b196c7d 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -52,6 +52,7 @@ types = ( make_type('Counter', []), make_type('Histogram', ['max', 'buckets']), ) +Shape = collections.namedtuple('Shape', 'max buckets') inst_map = dict((t[0].__name__, t[1]) for t in types) @@ -74,6 +75,10 @@ def dbl2u64(d): return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value +def u642dbl(d): + return ctypes.c_double.from_buffer(ctypes.c_ulonglong(d)).value + + def shift_works_until(mapped_bounds, shift_bits): for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])): a, b = ab @@ -91,22 +96,18 @@ def find_ideal_shift(mapped_bounds, max_size): table_size = mapped_bounds[n - 1] >> shift_bits if table_size > max_size: continue - if table_size > 65535: - continue if best is None: best = (shift_bits, n, table_size) elif best[1] < n: best = (shift_bits, n, table_size) - print(best) return best def gen_map_table(mapped_bounds, shift_data): + #print("gen_map_table(%s, %s)" % (mapped_bounds, shift_data)) tbl = [] cur = 0 - print(mapped_bounds) mapped_bounds = [x >> shift_data[0] for x in mapped_bounds] - print(mapped_bounds) for i in range(0, mapped_bounds[shift_data[1] - 1]): while i > mapped_bounds[cur]: cur += 1 @@ -123,7 +124,6 @@ def decl_static_table(values, type): for i, vp in enumerate(static_tables): if v == vp: return i - print("ADD TABLE: %s %r" % (type, values)) r = len(static_tables) static_tables.append(v) return r @@ -141,19 +141,30 @@ def type_for_uint_table(table): return 'uint64_t' -def gen_bucket_code(histogram): +def merge_cases(cases): + l = len(cases) + if l == 1: + return cases[0][1] + left_len = l // 2 + left = cases[0:left_len] + right = cases[left_len:] + return 'if (value < %d) {\n%s\n} else {\n%s\n}' % ( + left[-1][0], merge_cases(left), merge_cases(right)) + + +def gen_bucket_code(shape): bounds = [0, 1] done_trivial = False done_unmapped = False first_nontrivial = None first_unmapped = None - while len(bounds) < histogram.buckets + 1: - if len(bounds) == histogram.buckets: - nextb = int(histogram.max) + while len(bounds) < shape.buckets + 1: + if len(bounds) == shape.buckets: + nextb = int(shape.max) else: mul = math.pow( - float(histogram.max) / bounds[-1], - 1.0 / (histogram.buckets + 1 - len(bounds))) + float(shape.max) / bounds[-1], + 1.0 / (shape.buckets + 1 - len(bounds))) nextb = int(math.ceil(bounds[-1] * mul)) if nextb <= bounds[-1] + 1: nextb = bounds[-1] + 1 @@ -162,47 +173,55 @@ def gen_bucket_code(histogram): first_nontrivial = len(bounds) bounds.append(nextb) bounds_idx = decl_static_table(bounds, 'int') - if done_trivial: - first_nontrivial_code = dbl2u64(first_nontrivial) - code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds] - shift_data = find_ideal_shift(code_bounds[first_nontrivial:], - 256 * histogram.buckets) #print first_nontrivial, shift_data, bounds #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] - code = 'value = grpc_core::Clamp(value, 0, %d);\n' % histogram.max - map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) if first_nontrivial is None: - code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' % - histogram.name.upper()) - else: - code += 'if (value < %d) {\n' % first_nontrivial - code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' % - histogram.name.upper()) - code += 'return;\n' - code += '}' + return ('return grpc_core::Clamp(value, 0, %d);\n' % shape.max, + bounds_idx) + cases = [(0, 'return 0;'), (first_nontrivial, 'return value;')] + if done_trivial: first_nontrivial_code = dbl2u64(first_nontrivial) - if shift_data is not None: - map_table_idx = decl_static_table(map_table, - type_for_uint_table(map_table)) - code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n' - code += '_val.dbl = value;\n' - code += 'if (_val.uint < %dull) {\n' % ( - (map_table[-1] << shift_data[0]) + first_nontrivial_code) - code += 'int bucket = ' - code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % ( - map_table_idx, first_nontrivial_code, shift_data[0], - first_nontrivial) - code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx - code += 'bucket -= (_val.uint < _bkt.uint);\n' - code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper( - ) - code += 'return;\n' - code += '}\n' - code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, ' % histogram.name.upper( - ) - code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % ( - bounds_idx, histogram.buckets) - return (code, bounds_idx) + while True: + code = '' + first_nontrivial = u642dbl(first_nontrivial_code) + code_bounds_index = None + for i, b in enumerate(bounds): + if b > first_nontrivial: + code_bounds_index = i + break + code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds] + shift_data = find_ideal_shift(code_bounds[code_bounds_index:], + 65536) + if not shift_data: + break + map_table = gen_map_table(code_bounds[code_bounds_index:], + shift_data) + if not map_table: + break + if map_table[-1] < 8: + break + map_table_idx = decl_static_table( + [x + code_bounds_index for x in map_table], + type_for_uint_table(map_table)) + last_code = ( + (len(map_table) - 1) << shift_data[0]) + first_nontrivial_code + code += '// first_nontrivial_code=%d\n// last_code=%d [%f]\n' % ( + first_nontrivial_code, last_code, u642dbl(last_code)) + code += 'DblUint val;\n' + code += 'val.dbl = value;\n' + code += 'const int bucket = ' + code += 'grpc_stats_table_%d[((val.uint - %dull) >> %d)];\n' % ( + map_table_idx, first_nontrivial_code, shift_data[0]) + code += 'return bucket - (value < grpc_stats_table_%d[bucket]);' % bounds_idx + cases.append((int(u642dbl(last_code)) + 1, code)) + first_nontrivial_code = last_code + last = u642dbl(last_code) + 1 + for i, b in enumerate(bounds[:-2]): + if bounds[i + 1] < last: + continue + cases.append((bounds[i + 1], 'return %d;' % i)) + cases.append((None, 'return %d;' % (len(bounds) - 2))) + return (merge_cases(cases), bounds_idx) # utility: print a big comment block into a set of files @@ -215,6 +234,10 @@ def put_banner(files, banner): print(file=f) +shapes = set() +for histogram in inst_map['Histogram']: + shapes.add(Shape(max=histogram.max, buckets=histogram.buckets)) + with open('src/core/lib/debug/stats_data.h', 'w') as H: # copy-paste copyright notice from this file with open(sys.argv[0]) as my_source: @@ -241,8 +264,8 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H: print(file=H) print("#include ", file=H) print(file=H) - print("#include ", file=H) - print("#include \"src/core/lib/iomgr/exec_ctx.h\"", file=H) + print("// IWYU pragma: private, include \"src/core/lib/debug/stats.h\"", + file=H) print(file=H) for typename, instances in sorted(inst_map.items()): @@ -261,7 +284,6 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H: histo_start = [] histo_buckets = [] - histo_bucket_boundaries = [] print("typedef enum {", file=H) first_slot = 0 @@ -285,10 +307,16 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H: file=H) for histogram in inst_map['Histogram']: print( - "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" - % (histogram.name.upper(), histogram.name.lower()), + "#define GRPC_STATS_INC_%s(value) GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, grpc_core::BucketForHistogramValue_%d_%d(static_cast(value)))" + % (histogram.name.upper(), histogram.name.upper(), histogram.max, + histogram.buckets), file=H) - print("void grpc_stats_inc_%s(int x);" % histogram.name.lower(), file=H) + print("namespace grpc_core {", file=H) + for shape in shapes: + print("int BucketForHistogramValue_%d_%d(int value);" % + (shape.max, shape.buckets), + file=H) + print("}", file=H) for i, tbl in enumerate(static_tables): print("extern const %s grpc_stats_table_%d[%d];" % @@ -304,7 +332,7 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H: print("extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(inst_map['Histogram']), file=H) - print("extern void (*const grpc_stats_inc_histogram[%d])(int x);" % + print("extern int (*const grpc_stats_get_bucket[%d])(int value);" % len(inst_map['Histogram']), file=H) @@ -341,11 +369,14 @@ with open('src/core/lib/debug/stats_data.cc', 'w') as C: print(file=C) histo_code = [] - for histogram in inst_map['Histogram']: - code, bounds_idx = gen_bucket_code(histogram) - histo_bucket_boundaries.append(bounds_idx) + histo_bucket_boundaries = {} + for shape in shapes: + code, bounds_idx = gen_bucket_code(shape) + histo_bucket_boundaries[shape] = bounds_idx histo_code.append(code) + print("namespace { union DblUint { double dbl; uint64_t uint; }; }", file=C) + for typename, instances in sorted(inst_map.items()): print("const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (typename.lower(), typename.upper()), @@ -365,10 +396,12 @@ with open('src/core/lib/debug/stats_data.cc', 'w') as C: (tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1])), file=C) - for histogram, code in zip(inst_map['Histogram'], histo_code): - print(("void grpc_stats_inc_%s(int value) {%s}") % - (histogram.name.lower(), code), + print("namespace grpc_core {", file=C) + for shape, code in zip(shapes, histo_code): + print(("int BucketForHistogramValue_%d_%d(int value) {%s}") % + (shape.max, shape.buckets, code), file=C) + print("}", file=C) print( "const int grpc_stats_histo_buckets[%d] = {%s};" % @@ -379,11 +412,14 @@ with open('src/core/lib/debug/stats_data.cc', 'w') as C: file=C) print("const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (len(inst_map['Histogram']), ','.join( - 'grpc_stats_table_%d' % x for x in histo_bucket_boundaries)), + 'grpc_stats_table_%d' % + histo_bucket_boundaries[Shape(h.max, h.buckets)] + for h in inst_map['Histogram'])), file=C) - print("void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % + print("int (*const grpc_stats_get_bucket[%d])(int value) = {%s};" % (len(inst_map['Histogram']), ','.join( - 'grpc_stats_inc_%s' % histogram.name.lower() + 'grpc_core::BucketForHistogramValue_%d_%d' % + (histogram.max, histogram.buckets) for histogram in inst_map['Histogram'])), file=C) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 7a9f5bea81f..d51f551d025 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -39,9 +39,6 @@ def massage_qps_stats(scenario_result): stats[ "core_server_channels_created"] = massage_qps_stats_helpers.counter( core_stats, "server_channels_created") - stats[ - "core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter( - core_stats, "histogram_slow_lookups") stats["core_syscall_write"] = massage_qps_stats_helpers.counter( core_stats, "syscall_write") stats["core_syscall_read"] = massage_qps_stats_helpers.counter( diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index b654da9c97e..a6d2286060b 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -135,11 +135,6 @@ "name": "core_server_channels_created", "type": "INTEGER" }, - { - "mode": "NULLABLE", - "name": "core_histogram_slow_lookups", - "type": "INTEGER" - }, { "mode": "NULLABLE", "name": "core_syscall_write", @@ -427,11 +422,6 @@ "name": "core_server_channels_created", "type": "INTEGER" }, - { - "mode": "NULLABLE", - "name": "core_histogram_slow_lookups", - "type": "INTEGER" - }, { "mode": "NULLABLE", "name": "core_syscall_write",