Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Histogram support

Browse Source
pull/12357/head
Craig Tiller 8 years ago
parent e944c5d490
commit 640dfe499e
7 changed files with 625 additions and 44 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 43
      src/core/lib/debug/stats.c
  2. 9
      src/core/lib/debug/stats.h
  3. 308
      src/core/lib/debug/stats_data.c
  4. 149
      src/core/lib/debug/stats_data.h
  5. 4
      src/core/lib/debug/stats_data.yaml
  6. 4
      src/core/lib/iomgr/tcp_posix.c
  7. 152
      tools/codegen/core/gen_stats_data.py

43
src/core/lib/debug/stats.c
Unescape View File

@ -45,7 +45,31 @@ void grpc_stats_collect(grpc_stats_data *output) {
output->counters[i] += gpr_atm_no_barrier_load(
&grpc_stats_per_cpu_storage[core].counters[i]);
}
for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
output->histograms[i] += gpr_atm_no_barrier_load(
&grpc_stats_per_cpu_storage[core].histograms[i]);
}
}
}
int grpc_stats_histo_find_bucket_slow(grpc_exec_ctx *exec_ctx, double value,
const double *table, int table_size) {
GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx);
if (value < 0.0) return 0;
if (value >= table[table_size - 1]) return table_size - 1;
int a = 0;
int b = table_size - 1;
while (a < b) {
int c = a + ((b - a) / 2);
if (value < table[c]) {
b = c - 1;
} else if (value > table[c]) {
a = c + 1;
} else {
return c;
}
}
return a;
}
char *grpc_stats_data_as_json(const grpc_stats_data *data) {
@ -60,6 +84,25 @@ char *grpc_stats_data_as_json(const grpc_stats_data *data) {
gpr_strvec_add(&v, tmp);
is_first = false;
}
for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
gpr_asprintf(&tmp, "%s\"%s\": [", is_first ? "" : ", ",
grpc_stats_histogram_name[i]);
gpr_strvec_add(&v, tmp);
for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
gpr_asprintf(&tmp, "%s%" PRIdPTR, j == 0 ? "" : ",",
data->histograms[grpc_stats_histo_start[i] + j]);
gpr_strvec_add(&v, tmp);
}
gpr_asprintf(&tmp, "], \"%s_bkt\": [", grpc_stats_histogram_name[i]);
gpr_strvec_add(&v, tmp);
for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
gpr_asprintf(&tmp, "%s%lf", j == 0 ? "" : ",",
grpc_stats_histo_bucket_boundaries[i][j]);
gpr_strvec_add(&v, tmp);
}
gpr_strvec_add(&v, gpr_strdup("]"));
is_first = false;
}
gpr_strvec_add(&v, gpr_strdup("}"));
tmp = gpr_strvec_flatten(&v, NULL);
gpr_strvec_destroy(&v);

9
src/core/lib/debug/stats.h
Unescape View File

@ -25,6 +25,7 @@
typedef struct grpc_stats_data {
gpr_atm counters[GRPC_STATS_COUNTER_COUNT];
gpr_atm histograms[GRPC_STATS_HISTOGRAM_BUCKETS];
} grpc_stats_data;
extern grpc_stats_data *grpc_stats_per_cpu_storage;
@ -36,9 +37,17 @@ extern grpc_stats_data *grpc_stats_per_cpu_storage;
(gpr_atm_no_barrier_fetch_add( \
&GRPC_THREAD_STATS_DATA((exec_ctx))->counters[(ctr)], 1))
#define GRPC_STATS_INC_HISTOGRAM(exec_ctx, histogram, index) \
(gpr_atm_no_barrier_fetch_add( \
&GRPC_THREAD_STATS_DATA((exec_ctx)) \
->histograms[histogram##_FIRST_SLOT + (index)], \
1))
void grpc_stats_init(void);
void grpc_stats_shutdown(void);
void grpc_stats_collect(grpc_stats_data *output);
char *grpc_stats_data_as_json(const grpc_stats_data *data);
int grpc_stats_histo_find_bucket_slow(grpc_exec_ctx *exec_ctx, double value,
const double *table, int table_size);
#endif

308
src/core/lib/debug/stats_data.c
Unescape View File

@ -1,12 +1,12 @@
/*
* Copyright 2017 gRPC authors.
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -20,10 +20,300 @@
#include "src/core/lib/debug/stats_data.h"
const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
"client_calls_created",
"server_calls_created",
"syscall_write",
"syscall_read",
"syscall_poll",
"syscall_wait",
"client_calls_created", "server_calls_created", "syscall_write",
"syscall_read", "syscall_poll", "syscall_wait",
"histogram_slow_lookups",
};
const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = {
"tcp_write_size", "tcp_write_iov_size", "tcp_read_size", "client_latency",
};
const double grpc_stats_table_0[64] = {0,
1,
2,
3,
4,
5.17974600698,
6.70744217421,
8.68571170472,
11.2474451301,
14.5647272503,
18.8603969544,
24.4230164536,
31.6262554885,
40.9539926456,
53.032819969,
68.6741343683,
88.9286433193,
115.156946285,
149.120933174,
193.102139541,
250.055009057,
323.805358672,
419.307378404,
542.976429747,
703.119998467,
910.495751121,
1179.03418281,
1526.77440013,
1977.07590065,
2560.18775048,
3315.28056941,
4293.07782286,
5559.26317765,
7198.89281155,
9322.10907382,
12071.5393129,
15631.8768886,
20242.2879738,
26212.4775761,
33943.4940145,
43954.6693961,
56918.5058232,
73705.8508152,
95444.3966128,
123594.433061,
160046.942783,
207250.628202,
268376.403469,
347530.401059,
450029.801797,
582760.01722,
754637.218056,
977207.279236,
1265421.37565,
1638640.32942,
2121935.1758,
2747771.31348,
3558189.37227,
4607629.29828,
5966587.36485,
7726351.7696,
10005134.9318,
12956014.428,
16777216.0};
const uint8_t grpc_stats_table_1[87] = {
0, 1, 3, 3, 4, 6, 6, 7, 9, 9, 10, 12, 12, 13, 15, 15, 16, 18,
18, 19, 21, 21, 22, 24, 24, 25, 27, 27, 28, 30, 30, 31, 32, 34, 34, 36,
36, 37, 39, 39, 40, 42, 42, 43, 44, 46, 46, 47, 49, 49, 51, 51, 52, 53,
55, 55, 56, 58, 58, 59, 61, 61, 63, 63, 64, 65, 67, 67, 68, 70, 70, 71,
73, 73, 75, 75, 76, 77, 79, 79, 80, 82, 82, 83, 85, 85, 87};
const double grpc_stats_table_2[64] = {0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12.0020736244,
13.0954337532,
14.2883963681,
15.5900350167,
17.0102498252,
18.5598427974,
20.2505999737,
22.0953810747,
24.1082173107,
26.3044181014,
28.7006875181,
31.315251333,
34.1679956422,
37.2806181177,
40.6767930374,
44.3823513489,
48.4254771375,
52.8369219909,
57.6502388927,
62.902037423,
68.6322622068,
74.8844967285,
81.7062948236,
89.1495423679,
97.2708519163,
106.131993291,
115.800363399,
126.34949884,
137.859635225,
150.418317437,
164.121065485,
179.072101023,
195.38514005,
213.184257818,
232.604832535,
253.794575043,
276.914652285,
302.140913126,
329.665225843,
359.696937452,
392.464465978,
428.217037783,
467.226583154,
509.78980457,
556.230433401,
606.901692163,
662.1889811,
722.512809492,
788.331994007,
860.147148411,
938.504491184,
1024.0};
const uint8_t grpc_stats_table_3[52] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52};
const double grpc_stats_table_4[128] = {0,
1,
2,
3,
4,
5,
6.04764436132,
7.3148004642,
8.8474623563,
10.7012611662,
12.9434843502,
15.6555180292,
18.9358010666,
22.9033981095,
27.7023212864,
33.5067574247,
40.5273905211,
49.0190489527,
59.2899549993,
71.7129124069,
86.7388380702,
104.913128993,
126.895458596,
153.483720931,
185.642991889,
224.540558623,
271.588288649,
328.493876489,
397.322827976,
480.573432046,
581.267441303,
703.059752763,
850.371069894,
1028.54836117,
1244.05893936,
1504.72520595,
1820.00858143,
2201.352927,
2662.59992325,
3220.49148246,
3895.27743092,
4711.45051817,
5698.63543198,
6892.66408748,
8336.87622063,
10083.6924933,
12196.5172097,
14752.0397062,
17843.0179495,
21581.6453782,
26103.6231959,
31573.0859261,
38188.5590141,
46190.1647178,
55868.3378408,
67574.3676638,
81733.1487144,
98858.6031911,
119572.334831,
144626.191302,
174929.554066,
211582.346255,
255914.956657,
309536.528921,
374393.36875,
452839.589087,
547722.557505,
662486.247293,
801296.243578,
969190.941845,
1172264.4269,
1417887.67026,
1714976.07481,
2074313.07772,
2508941.55762,
3034637.25277,
3670481.37407,
4439553.19704,
5369767.77177,
6494889.15731,
7855755.95793,
9501763.64457,
11492657.4655,
13900701.0236,
16813299.2328,
20336170.86,
24597185.8065,
29751006.4094,
35984701.2311,
43524535.0988,
52644181.8539,
63674657.909,
77016337.1725,
93153483.4461,
112671827.78,
136279828.791,
164834387.63,
199371950.98,
241146131.03,
291673207.915,
352787166.24,
426706263.331,
516113545.475,
624254234.618,
755053520.404,
913259033.033,
1104613168.31,
1336061519.75,
1616004983.26,
1954604684.98,
2364150800.34,
2859508651.4,
3458658274.36,
4183347042.13,
5059879030.16,
6120069777.14,
7402401095.72,
8953417849.36,
10829417394.2,
13098493008.1,
15843005476.5,
19162572547.2,
23177684762.8,
28034078912.8,
33908027852.3,
41012738688.9,
49606091574.9,
60000000000.0};
const uint16_t grpc_stats_table_5[265] = {
0, 2, 2, 4, 4, 6, 6, 8, 8, 11, 11, 11, 13, 13, 15,
15, 17, 17, 20, 20, 20, 22, 22, 24, 24, 26, 26, 28, 28, 30,
30, 33, 33, 33, 35, 35, 37, 37, 39, 39, 42, 42, 42, 44, 44,
46, 46, 48, 48, 51, 51, 51, 53, 53, 54, 57, 57, 57, 60, 60,
60, 61, 63, 63, 66, 66, 66, 68, 68, 70, 70, 72, 72, 75, 75,
75, 77, 77, 79, 79, 81, 81, 84, 84, 84, 85, 87, 87, 90, 90,
90, 92, 92, 94, 94, 96, 96, 99, 99, 99, 101, 101, 103, 103, 105,
105, 108, 108, 108, 109, 112, 112, 112, 114, 114, 116, 116, 118, 118, 120,
120, 123, 123, 123, 125, 125, 127, 127, 129, 129, 132, 132, 132, 134, 134,
136, 136, 138, 138, 140, 140, 142, 142, 145, 145, 145, 147, 147, 149, 149,
151, 151, 154, 154, 154, 156, 156, 158, 158, 160, 160, 162, 162, 165, 165,
165, 166, 169, 169, 169, 172, 172, 172, 173, 175, 175, 178, 178, 178, 180,
180, 182, 182, 184, 184, 187, 187, 187, 189, 189, 191, 191, 193, 193, 196,
196, 196, 197, 199, 199, 202, 202, 202, 204, 204, 206, 206, 208, 208, 211,
211, 211, 213, 213, 215, 215, 217, 217, 220, 220, 220, 221, 224, 224, 224,
226, 226, 228, 228, 230, 230, 232, 232, 235, 235, 235, 237, 237, 239, 239,
241, 241, 244, 244, 244, 246, 246, 248, 248, 250, 250, 252, 252, 254, 254,
257, 257, 257, 259, 259, 261, 261, 263, 263, 265};
const int grpc_stats_histo_buckets[4] = {64, 64, 64, 128};
const int grpc_stats_histo_start[4] = {0, 64, 128, 192};
const double *const grpc_stats_histo_bucket_boundaries[4] = {
grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0,
grpc_stats_table_4};

149
src/core/lib/debug/stats_data.h
Unescape View File

@ -1,12 +1,12 @@
/*
* Copyright 2017 gRPC authors.
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -21,6 +21,8 @@
#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H
#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H
#include <inttypes.h>
typedef enum {
GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED,
GRPC_STATS_COUNTER_SERVER_CALLS_CREATED,
@ -28,23 +30,144 @@ typedef enum {
GRPC_STATS_COUNTER_SYSCALL_READ,
GRPC_STATS_COUNTER_SYSCALL_POLL,
GRPC_STATS_COUNTER_SYSCALL_WAIT,
GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS,
GRPC_STATS_COUNTER_COUNT
} grpc_stats_counters;
extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
typedef enum {
GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE,
GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
GRPC_STATS_HISTOGRAM_CLIENT_LATENCY,
GRPC_STATS_HISTOGRAM_COUNT
} grpc_stats_histograms;
#define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
#define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED)
#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
#define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL)
#define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT)
#define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,None)
#define GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, value) GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,None)
#define GRPC_STATS_INC_CLIENT_LATENCY(exec_ctx, value) GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CLIENT_LATENCY,None)
extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT];
typedef enum {
GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0,
GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64,
GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64,
GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64,
GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128,
GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64,
GRPC_STATS_HISTOGRAM_CLIENT_LATENCY_FIRST_SLOT = 192,
GRPC_STATS_HISTOGRAM_CLIENT_LATENCY_BUCKETS = 128,
GRPC_STATS_HISTOGRAM_BUCKETS = 320
} grpc_stats_histogram_constants;
#define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
#define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED)
#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
#define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL)
#define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT)
#define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS)
#define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \
do { \
double _hist_val = (double)(value); \
if (_hist_val < 0) _hist_val = 0; \
uint64_t _hist_idx = *(uint64_t *)&_hist_val; \
gpr_log(GPR_DEBUG, "tcp_write_size %lf %" PRId64 " %" PRId64, _hist_val, \
_hist_idx, 4715268809856909312ull); \
if (_hist_val < 5.000000) { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, (int)_hist_val); \
} else { \
if (_hist_idx < 4715268809856909312ull) { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \
grpc_stats_table_1[((_hist_idx - 4617315517961601024ull) >> 50)]); \
} else { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \
grpc_stats_histo_find_bucket_slow((exec_ctx), (value), \
grpc_stats_table_0, 64)); \
} \
} \
} while (false)
#define GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(exec_ctx, value) \
do { \
double _hist_val = (double)(value); \
if (_hist_val < 0) _hist_val = 0; \
uint64_t _hist_idx = *(uint64_t *)&_hist_val; \
gpr_log(GPR_DEBUG, "tcp_write_iov_size %lf %" PRId64 " %" PRId64, \
_hist_val, _hist_idx, 4652218415073722368ull); \
if (_hist_val < 12.000000) { \
GRPC_STATS_INC_HISTOGRAM((exec_ctx), \
GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \
(int)_hist_val); \
} else { \
if (_hist_idx < 4652218415073722368ull) { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \
grpc_stats_table_3[((_hist_idx - 4622945017495814144ull) >> 49)]); \
} else { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \
grpc_stats_histo_find_bucket_slow((exec_ctx), (value), \
grpc_stats_table_2, 64)); \
} \
} \
} while (false)
#define GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, value) \
do { \
double _hist_val = (double)(value); \
if (_hist_val < 0) _hist_val = 0; \
uint64_t _hist_idx = *(uint64_t *)&_hist_val; \
gpr_log(GPR_DEBUG, "tcp_read_size %lf %" PRId64 " %" PRId64, _hist_val, \
_hist_idx, 4715268809856909312ull); \
if (_hist_val < 5.000000) { \
GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \
(int)_hist_val); \
} else { \
if (_hist_idx < 4715268809856909312ull) { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \
grpc_stats_table_1[((_hist_idx - 4617315517961601024ull) >> 50)]); \
} else { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \
grpc_stats_histo_find_bucket_slow((exec_ctx), (value), \
grpc_stats_table_0, 64)); \
} \
} \
} while (false)
#define GRPC_STATS_INC_CLIENT_LATENCY(exec_ctx, value) \
do { \
double _hist_val = (double)(value); \
if (_hist_val < 0) _hist_val = 0; \
uint64_t _hist_idx = *(uint64_t *)&_hist_val; \
gpr_log(GPR_DEBUG, "client_latency %lf %" PRId64 " %" PRId64, _hist_val, \
_hist_idx, 4767623155525091328ull); \
if (_hist_val < 6.000000) { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_CLIENT_LATENCY, (int)_hist_val); \
} else { \
if (_hist_idx < 4767623155525091328ull) { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_CLIENT_LATENCY, \
grpc_stats_table_5[((_hist_idx - 4618441417868443648ull) >> 49)]); \
} else { \
GRPC_STATS_INC_HISTOGRAM( \
(exec_ctx), GRPC_STATS_HISTOGRAM_CLIENT_LATENCY, \
grpc_stats_histo_find_bucket_slow((exec_ctx), (value), \
grpc_stats_table_4, 128)); \
} \
} \
} while (false)
extern const double grpc_stats_table_0[64];
extern const uint8_t grpc_stats_table_1[87];
extern const double grpc_stats_table_2[64];
extern const uint8_t grpc_stats_table_3[52];
extern const double grpc_stats_table_4[128];
extern const uint16_t grpc_stats_table_5[265];
extern const int grpc_stats_histo_buckets[4];
extern const int grpc_stats_histo_start[4];
extern const double *const grpc_stats_histo_bucket_boundaries[4];
#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */

4
src/core/lib/debug/stats_data.yaml
Unescape View File

@ -7,9 +7,13 @@
- counter: syscall_read
- counter: syscall_poll
- counter: syscall_wait
- counter: histogram_slow_lookups
- histogram: tcp_write_size
max: 16777216 # 16 meg max write tracked
buckets: 64
- histogram: tcp_write_iov_size
max: 1024
buckets: 64
- histogram: tcp_read_size
max: 16777216
buckets: 64

4
src/core/lib/iomgr/tcp_posix.c
Unescape View File

@ -287,6 +287,7 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
TCP_UNREF(exec_ctx, tcp, "read");
} else {
GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, read_bytes);
add_to_estimate(tcp, (size_t)read_bytes);
GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
if ((size_t)read_bytes < tcp->incoming_buffer->length) {
@ -403,6 +404,9 @@ static bool tcp_flush(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
msg.msg_controllen = 0;
msg.msg_flags = 0;
GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, sending_length);
GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(exec_ctx, iov_size);
GPR_TIMER_BEGIN("sendmsg", 1);
do {
/* TODO(klempner): Cork if this is a partial write */

152
tools/codegen/core/gen_stats_data.py
Unescape View File

@ -47,17 +47,60 @@ for attr in attrs:
def dbl2u64(d):
return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value
def shift_works(mapped_bounds, shift_bits):
for a, b in zip(mapped_bounds, mapped_bounds[1:]):
def shift_works_until(mapped_bounds, shift_bits):
for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])):
a, b = ab
if (a >> shift_bits) == (b >> shift_bits):
return False
return True
return i
return len(mapped_bounds)
def find_max_shift(mapped_bounds):
def find_ideal_shift(mapped_bounds, max_size):
best = None
for shift_bits in reversed(range(0,64)):
if shift_works(mapped_bounds, shift_bits):
return shift_bits
return -1
n = shift_works_until(mapped_bounds, shift_bits)
if n == 0: continue
table_size = mapped_bounds[n-1] >> shift_bits
if table_size > max_size: continue
if table_size > 65535: continue
if best is None:
best = (shift_bits, n, table_size)
elif best[1] < n:
best = (shift_bits, n, table_size)
print best
return best
def gen_map_table(mapped_bounds, shift_data):
tbl = []
cur = 0
mapped_bounds = [x >> shift_data[0] for x in mapped_bounds]
for i in range(0, mapped_bounds[shift_data[1]-1]):
while i > mapped_bounds[cur]:
cur += 1
tbl.append(mapped_bounds[cur])
return tbl
static_tables = []
def decl_static_table(values, type):
global static_tables
v = (type, values)
for i, vp in enumerate(static_tables):
if v == vp: return i
print "ADD TABLE: %s %r" % (type, values)
r = len(static_tables)
static_tables.append(v)
return r
def type_for_uint_table(table):
mv = max(table)
if mv < 2**8:
return 'uint8_t'
elif mv < 2**16:
return 'uint16_t'
elif mv < 2**32:
return 'uint32_t'
else:
return 'uint64_t'
def gen_bucket_code(histogram):
bounds = [0, 1]
@ -75,10 +118,41 @@ def gen_bucket_code(histogram):
done_trivial = True
first_nontrivial = len(bounds)
bounds.append(nextb)
bounds_idx = decl_static_table(bounds, 'double')
if done_trivial:
code_bounds = [dbl2u64(x - first_nontrivial) for x in bounds]
shift_bits = find_max_shift(code_bounds[first_nontrivial:])
print first_nontrivial, shift_bits, bounds, [hex(x >> shift_bits) for x in code_bounds[first_nontrivial:]]
first_nontrivial_code = dbl2u64(first_nontrivial)
code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds]
shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
print first_nontrivial, shift_data, bounds
if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
code = 'do {\\\n'
code += 'double _hist_val = (double)(value);\\\n'
code += 'if (_hist_val < 0) _hist_val = 0;\\\n'
code += 'uint64_t _hist_idx = *(uint64_t*)&_hist_val;\\\n'
map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
code += 'gpr_log(GPR_DEBUG, "' + histogram.name + ' %lf %"PRId64 " %"PRId64, _hist_val, _hist_idx, ' + str((map_table[-1] << shift_data[0]) + first_nontrivial_code) + 'ull);\\\n'
if first_nontrivial is None:
code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, (int)_hist_val);\\\n'
% histogram.name.upper())
else:
code += 'if (_hist_val < %f) {\\\n' % first_nontrivial
code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, (int)_hist_val);\\\n'
% histogram.name.upper())
code += '} else {'
first_nontrivial_code = dbl2u64(first_nontrivial)
if shift_data is not None:
map_table_idx = decl_static_table(map_table, type_for_uint_table(map_table))
code += 'if (_hist_idx < %dull) {\\\n' % ((map_table[-1] << shift_data[0]) + first_nontrivial_code)
code += 'GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, ' % histogram.name.upper()
code += 'grpc_stats_table_%d[((_hist_idx - %dull) >> %d)]);\\\n' % (map_table_idx, first_nontrivial_code, shift_data[0])
code += '} else {\\\n'
code += 'GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, '% histogram.name.upper()
code += 'grpc_stats_histo_find_bucket_slow((exec_ctx), (value), grpc_stats_table_%d, %d));\\\n' % (bounds_idx, len(bounds))
if shift_data is not None:
code += '}'
code += '}'
code += '} while (false)'
return (code, bounds_idx)
# utility: print a big comment block into a set of files
def put_banner(files, banner):
@ -110,6 +184,8 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H:
print >>H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
print >>H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
print >>H
print >>H, "#include <inttypes.h>"
print >>H
for typename, instances in sorted(inst_map.items()):
print >>H, "typedef enum {"
@ -117,21 +193,41 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H:
print >>H, " GRPC_STATS_%s_%s," % (typename.upper(), inst.name.upper())
print >>H, " GRPC_STATS_%s_COUNT" % (typename.upper())
print >>H, "} grpc_stats_%ss;" % (typename.lower())
print >>H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % (
typename.lower(), typename.upper())
histo_start = []
histo_buckets = []
histo_bucket_boundaries = []
print >>H, "typedef enum {"
first_slot = 0
for histogram in inst_map['Histogram']:
histo_start.append(first_slot)
histo_buckets.append(histogram.buckets)
print >>H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (histogram.name.upper(), first_slot)
print >>H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (histogram.name.upper(), histogram.buckets)
first_slot += histogram.buckets
print >>H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot
print >>H, "} grpc_stats_histogram_constants;"
for ctr in inst_map['Counter']:
print >>H, ("#define GRPC_STATS_INC_%s(exec_ctx) " +
"GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_%s)") % (
ctr.name.upper(), ctr.name.upper())
for histogram in inst_map['Histogram']:
print >>H, ("#define GRPC_STATS_INC_%s(exec_ctx, value) " +
"GRPC_STATS_INC_HISTOGRAM((exec_ctx), " +
"GRPC_STATS_HISTOGRAM_%s," +
"%s)") % (
histogram.name.upper(),
code, bounds_idx = gen_bucket_code(histogram)
histo_bucket_boundaries.append(bounds_idx)
print >>H, ("#define GRPC_STATS_INC_%s(exec_ctx, value) %s") % (
histogram.name.upper(),
gen_bucket_code(histogram))
code)
print >>H, "extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];"
for i, tbl in enumerate(static_tables):
print >>H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, len(tbl[1]))
print >>H, "extern const int grpc_stats_histo_buckets[%d];" % len(inst_map['Histogram'])
print >>H, "extern const int grpc_stats_histo_start[%d];" % len(inst_map['Histogram'])
print >>H, "extern const double *const grpc_stats_histo_bucket_boundaries[%d];" % len(inst_map['Histogram'])
print >>H
print >>H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
@ -156,7 +252,19 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C:
print >>C, "#include \"src/core/lib/debug/stats_data.h\""
print >>C, "const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {";
for ctr in inst_map['Counter']:
print >>C, " \"%s\"," % ctr.name
print >>C, "};"
for typename, instances in sorted(inst_map.items()):
print >>C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (
typename.lower(), typename.upper())
for inst in instances:
print >>C, " \"%s\"," % inst.name
print >>C, "};"
for i, tbl in enumerate(static_tables):
print >>C, "const %s grpc_stats_table_%d[%d] = {%s};" % (
tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1]))
print >>C, "const int grpc_stats_histo_buckets[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets))
print >>C, "const int grpc_stats_histo_start[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start))
print >>C, "const double *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))

Write Preview
Loading…
Cancel
Save