From fa6b606898b18c22cdcbaa006338fe1d57d8f93f Mon Sep 17 00:00:00 2001 From: Dan Born Date: Fri, 8 Jan 2016 21:01:59 -0800 Subject: [PATCH 01/42] Make grpc_tcp_listener private. --- src/core/iomgr/tcp_server.h | 52 ++++--- src/core/iomgr/tcp_server_posix.c | 133 ++++++++++++------ src/core/iomgr/tcp_server_windows.c | 120 ++++++++++------ src/core/security/server_secure_chttp2.c | 29 ++-- src/core/surface/server_chttp2.c | 15 +- .../set_initial_connect_string_test.c | 5 +- test/core/iomgr/tcp_server_posix_test.c | 54 ++++--- test/core/util/reconnect_server.c | 5 +- test/core/util/test_tcp_server.c | 11 +- 9 files changed, 269 insertions(+), 155 deletions(-) diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h index 3294e137974..92e504d20be 100644 --- a/src/core/iomgr/tcp_server.h +++ b/src/core/iomgr/tcp_server.h @@ -39,23 +39,25 @@ /* Forward decl of grpc_tcp_server */ typedef struct grpc_tcp_server grpc_tcp_server; -/* Forward decl of grpc_tcp_listener */ -typedef struct grpc_tcp_listener grpc_tcp_listener; - -/* Called for newly connected TCP connections. */ +/* Called for newly connected TCP connections. Callee owns a ref on + from_server. */ typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg, - grpc_endpoint *ep); + grpc_endpoint *ep, + grpc_tcp_server *from_server, + unsigned port_index, unsigned fd_index); -/* Create a server, initially not bound to any ports */ -grpc_tcp_server *grpc_tcp_server_create(void); +/* Create a server, initially not bound to any ports. The caller owns one ref. + If shutdown_complete is not NULL, it will be used by + grpc_tcp_server_unref(). */ +grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete); /* Start listening to bound ports */ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, grpc_pollset **pollsets, size_t pollset_count, grpc_tcp_server_cb on_accept_cb, void *cb_arg); -/* Add a port to the server, returning the newly created listener on success, - or a null pointer on failure. +/* Add a port to the server, returning the newly allocated port on success, or + -1 on failure. The :: and 0.0.0.0 wildcard addresses are treated identically, accepting both IPv4 and IPv6 connections, but :: is the preferred style. This usually @@ -63,21 +65,29 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, but not dualstack sockets. */ /* TODO(ctiller): deprecate this, and make grpc_tcp_server_add_ports to handle all of the multiple socket port matching logic in one place */ -grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, - const void *addr, size_t addr_len); +int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, + size_t addr_len); + +/* Number of fds at the given port_index, or 0 if port_index is out of + bounds. */ +unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, unsigned port_index); -/* Returns the file descriptor of the Nth listening socket on this server, - or -1 if the index is out of bounds. +/* Returns the file descriptor of the Mth (fd_index) listening socket of the Nth + (port_index) call to add_port() on this server, or -1 if the indices are out + of bounds. The file descriptor remains owned by the server, and will be + cleaned up when grpc_tcp_server_destroy is called. */ +int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index, + unsigned fd_index); - The file descriptor remains owned by the server, and will be cleaned - up when grpc_tcp_server_destroy is called. */ -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index); +/* Ref s and return s. */ +grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s); -void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, - grpc_closure *closure); +/* Set or reset the shutdown_complete closure. shutdown_complete may be NULL. */ +void grpc_tcp_server_set_shutdown_complete(grpc_tcp_server *s, + grpc_closure *shutdown_complete); -int grpc_tcp_listener_get_port(grpc_tcp_listener *listener); -void grpc_tcp_listener_ref(grpc_tcp_listener *listener); -void grpc_tcp_listener_unref(grpc_tcp_listener *listener); +/* If the recount drops to zero, delete s, and call (exec_ctx==NULL) or enqueue + a call (exec_ctx!=NULL) to shutdown_complete. */ +void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s); #endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */ diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index 49e83cf6aeb..f1119f689ad 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -73,6 +73,7 @@ static gpr_once s_init_max_accept_queue_size; static int s_max_accept_queue_size; /* one listening port */ +typedef struct grpc_tcp_listener grpc_tcp_listener; struct grpc_tcp_listener { int fd; grpc_fd *emfd; @@ -84,9 +85,10 @@ struct grpc_tcp_listener { } addr; size_t addr_len; int port; + unsigned port_index; + unsigned fd_index; grpc_closure read_closure; grpc_closure destroyed_closure; - gpr_refcount refs; struct grpc_tcp_listener *next; /* When we add a listener, more than one can be created, mainly because of IPv6. A sibling will still be in the normal list, but will be flagged @@ -106,6 +108,7 @@ static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) { /* the overall server */ struct grpc_tcp_server { + gpr_refcount refs; /* Called whenever accept() succeeds on a server port. */ grpc_tcp_server_cb on_accept_cb; void *on_accept_cb_arg; @@ -122,6 +125,7 @@ struct grpc_tcp_server { /* linked list of server ports */ grpc_tcp_listener *head; + grpc_tcp_listener *tail; unsigned nports; /* shutdown callback */ @@ -133,28 +137,33 @@ struct grpc_tcp_server { size_t pollset_count; }; -grpc_tcp_server *grpc_tcp_server_create(void) { +grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); + gpr_ref_init(&s->refs, 1); gpr_mu_init(&s->mu); s->active_ports = 0; s->destroyed_ports = 0; s->shutdown = 0; + s->shutdown_complete = shutdown_complete; s->on_accept_cb = NULL; s->on_accept_cb_arg = NULL; s->head = NULL; + s->tail = NULL; s->nports = 0; return s; } static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { - grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); + if (s->shutdown_complete != NULL) { + grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); + } gpr_mu_destroy(&s->mu); while (s->head) { grpc_tcp_listener *sp = s->head; s->head = sp->next; - grpc_tcp_listener_unref(sp); + gpr_free(sp); } gpr_free(s); @@ -203,15 +212,13 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { } } -void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, - grpc_closure *closure) { +static void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, + grpc_tcp_server *s) { gpr_mu_lock(&s->mu); GPR_ASSERT(!s->shutdown); s->shutdown = 1; - s->shutdown_complete = closure; - /* shutdown all fd's */ if (s->active_ports) { grpc_tcp_listener *sp; @@ -355,7 +362,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { } sp->server->on_accept_cb( exec_ctx, sp->server->on_accept_cb_arg, - grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str)); + grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str), + grpc_tcp_server_ref(sp->server), sp->port_index, sp->fd_index); gpr_free(name); gpr_free(addr_str); @@ -375,7 +383,9 @@ error: static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd, const struct sockaddr *addr, - size_t addr_len) { + size_t addr_len, + unsigned port_index, + unsigned fd_index) { grpc_tcp_listener *sp = NULL; int port; char *addr_str; @@ -389,17 +399,23 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd, s->nports++; GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); sp = gpr_malloc(sizeof(grpc_tcp_listener)); - sp->next = s->head; - s->head = sp; + sp->next = NULL; + if (s->head == NULL) { + s->head = sp; + } else { + s->tail->next = sp; + } + s->tail = sp; sp->server = s; sp->fd = fd; sp->emfd = grpc_fd_create(fd, name); memcpy(sp->addr.untyped, addr, addr_len); sp->addr_len = addr_len; sp->port = port; + sp->port_index = port_index; + sp->fd_index = fd_index; sp->is_sibling = 0; sp->sibling = NULL; - gpr_ref_init(&sp->refs, 1); GPR_ASSERT(sp->emfd); gpr_mu_unlock(&s->mu); gpr_free(addr_str); @@ -409,8 +425,8 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd, return sp; } -grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, - const void *addr, size_t addr_len) { +int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, + size_t addr_len) { grpc_tcp_listener *sp; grpc_tcp_listener *sp2 = NULL; int fd; @@ -423,7 +439,11 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, struct sockaddr_storage sockname_temp; socklen_t sockname_len; int port; - + unsigned port_index = 0; + unsigned fd_index = 0; + if (s->tail != NULL) { + port_index = s->tail->port_index + 1; + } if (((struct sockaddr *)addr)->sa_family == AF_UNIX) { unlink_if_unix_domain_socket(addr); } @@ -462,11 +482,13 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, addr = (struct sockaddr *)&wild6; addr_len = sizeof(wild6); fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode); - sp = add_socket_to_server(s, fd, addr, addr_len); + sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index); if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) { goto done; } - + if (sp != NULL) { + ++fd_index; + } /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */ if (port == 0 && sp != NULL) { grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port); @@ -485,20 +507,47 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, addr = (struct sockaddr *)&addr4_copy; addr_len = sizeof(addr4_copy); } - sp = add_socket_to_server(s, fd, addr, addr_len); - if (sp != NULL) sp->sibling = sp2; - if (sp2 != NULL) sp2->is_sibling = 1; + sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index); + if (sp2 != NULL) { + if (sp != NULL) { + sp->sibling = sp2; + } + sp2->is_sibling = 1; + } done: gpr_free(allocated_addr); - return sp; + if (sp != NULL) { + return sp->port; + } else { + return -1; + } +} + +unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, unsigned port_index) { + unsigned num_fds = 0; + grpc_tcp_listener *sp; + for (sp = s->head; sp && port_index != 0; sp = sp->next) { + if (!sp->is_sibling) { + --port_index; + } + } + for (; sp; sp = sp->sibling, ++num_fds) + ; + return num_fds; } -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) { +int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index, + unsigned fd_index) { grpc_tcp_listener *sp; - for (sp = s->head; sp && port_index != 0; sp = sp->next, port_index--) + for (sp = s->head; sp && port_index != 0; sp = sp->next) { + if (!sp->is_sibling) { + --port_index; + } + } + for (; sp && fd_index != 0; sp = sp->sibling, --fd_index) ; - if (port_index == 0 && sp) { + if (sp) { return sp->fd; } else { return -1; @@ -531,31 +580,25 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, gpr_mu_unlock(&s->mu); } -int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) { - if (listener != NULL) { - grpc_tcp_listener *sp = listener; - return sp->port; - } else { - return 0; - } +grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) { + gpr_ref(&s->refs); + return s; } -void grpc_tcp_listener_ref(grpc_tcp_listener *listener) { - grpc_tcp_listener *sp = listener; - gpr_ref(&sp->refs); +void grpc_tcp_server_set_shutdown_complete(grpc_tcp_server *s, + grpc_closure *shutdown_complete) { + s->shutdown_complete = shutdown_complete; } -void grpc_tcp_listener_unref(grpc_tcp_listener *listener) { - grpc_tcp_listener *sp = listener; - if (sp->is_sibling) return; - if (gpr_unref(&sp->refs)) { - grpc_tcp_listener *sibling = sp->sibling; - while (sibling) { - sp = sibling; - sibling = sp->sibling; - gpr_free(sp); +void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { + if (gpr_unref(&s->refs)) { + if (exec_ctx == NULL) { + grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_tcp_server_destroy(&local_exec_ctx, s); + grpc_exec_ctx_finish(&local_exec_ctx); + } else { + grpc_tcp_server_destroy(exec_ctx, s); } - gpr_free(listener); } } diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index d38fd8860a1..0435f5005bb 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -65,19 +65,20 @@ struct grpc_tcp_listener { grpc_winsocket *socket; /* The actual TCP port number. */ int port; + unsigned port_index; grpc_tcp_server *server; /* The cached AcceptEx for that port. */ LPFN_ACCEPTEX AcceptEx; int shutting_down; /* closure for socket notification of accept being ready */ grpc_closure on_accept; - gpr_refcount refs; /* linked list */ struct grpc_tcp_listener *next; }; /* the overall server */ struct grpc_tcp_server { + gpr_refcount refs; /* Called whenever accept() succeeds on a server port. */ grpc_tcp_server_cb on_accept_cb; void *on_accept_cb_arg; @@ -89,6 +90,7 @@ struct grpc_tcp_server { /* linked list of server ports */ grpc_tcp_listener *head; + grpc_tcp_listener *tail; /* shutdown callback */ grpc_closure *shutdown_complete; @@ -96,21 +98,23 @@ struct grpc_tcp_server { /* Public function. Allocates the proper data structures to hold a grpc_tcp_server. */ -grpc_tcp_server *grpc_tcp_server_create(void) { +grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); + gpr_ref_init(&s->refs, 1); gpr_mu_init(&s->mu); s->active_ports = 0; s->on_accept_cb = NULL; s->on_accept_cb_arg = NULL; s->head = NULL; - s->shutdown_complete = NULL; + s->tail = NULL; + s->shutdown_complete = shutdown_complete; return s; } -static void dont_care_about_shutdown_completion(void *arg) {} - static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { - grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); + if (s->shutdown_complete != NULL) { + grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); + } /* Now that the accepts have been aborted, we can destroy the sockets. The IOCP won't get notified on these, so we can flag them as already @@ -125,15 +129,17 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { gpr_free(s); } -/* Public function. Stops and destroys a grpc_tcp_server. */ -void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, - grpc_closure *shutdown_complete) { +grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) { + gpr_ref(&s->refs); + return s; +} + +static void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, + grpc_tcp_server *s) { int immediately_done = 0; grpc_tcp_listener *sp; gpr_mu_lock(&s->mu); - s->shutdown_complete = shutdown_complete; - /* First, shutdown all fd's. This will queue abortion calls for all of the pending accepts due to the normal operation mechanism. */ if (s->active_ports == 0) { @@ -150,6 +156,23 @@ void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, } } +void grpc_tcp_server_set_shutdown_complete(grpc_tcp_server *s, + grpc_closure *shutdown_complete) { + s->shutdown_complete = shutdown_complete; +} + +void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { + if (gpr_unref(&s->refs)) { + if (exec_ctx == NULL) { + grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_tcp_server_destroy(&local_exec_ctx, s); + grpc_exec_ctx_finish(&local_exec_ctx); + } else { + grpc_tcp_server_destroy(exec_ctx, s); + } + } +} + /* Prepare (bind) a recently-created socket for listening. */ static int prepare_socket(SOCKET sock, const struct sockaddr *addr, size_t addr_len) { @@ -343,7 +366,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { /* The only time we should call our callback, is where we successfully managed to accept a connection, and created an endpoint. */ - if (ep) sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep); + if (ep) sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, + sp->server, sp->port_index, 0); /* As we were notified from the IOCP of one and exactly one accept, the former socked we created has now either been destroy or assigned to the new connection. We need to create a new one for the next @@ -353,7 +377,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock, const struct sockaddr *addr, - size_t addr_len) { + size_t addr_len, + unsigned port_index) { grpc_tcp_listener *sp = NULL; int port; int status; @@ -382,14 +407,20 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock, gpr_mu_lock(&s->mu); GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); sp = gpr_malloc(sizeof(grpc_tcp_listener)); - sp->next = s->head; - s->head = sp; + sp->next = NULL; + if (s->head == NULL) { + s->head = sp; + } else { + s->tail->next = sp; + } + s->tail = sp; sp->server = s; sp->socket = grpc_winsocket_create(sock, "listener"); sp->shutting_down = 0; sp->AcceptEx = AcceptEx; sp->new_socket = INVALID_SOCKET; sp->port = port; + sp->port_index = port_index; gpr_ref_init(&sp->refs, 1); grpc_closure_init(&sp->on_accept, on_accept, sp); GPR_ASSERT(sp->socket); @@ -399,8 +430,8 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock, return sp; } -grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, - const void *addr, size_t addr_len) { +int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, + size_t addr_len) { grpc_tcp_listener *sp; SOCKET sock; struct sockaddr_in6 addr6_v4mapped; @@ -409,6 +440,10 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, struct sockaddr_storage sockname_temp; socklen_t sockname_len; int port; + unsigned port_index = 0; + if (s->tail != NULL) { + port_index = s->tail->port_index + 1; + } /* Check if this is a wildcard port, and if so, try to keep the port the same as some previously created listener. */ @@ -450,17 +485,37 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, gpr_free(utf8_message); } - sp = add_socket_to_server(s, sock, addr, addr_len); + sp = add_socket_to_server(s, sock, addr, addr_len, port_index); gpr_free(allocated_addr); - return sp; + if (sp) { + return sp->port; + } else { + return -1; + } +} + +unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, int port_index) { + grpc_tcp_listener *sp; + for (sp = s->head; sp && port_index != 0; sp = sp->next, --port_index) + ; + if (sp) { + return 1; + } else { + return 0; + } } -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) { +int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index, + unsigned fd_index) { grpc_tcp_listener *sp; - for (sp = s->head; sp && port_index != 0; sp = sp->next, port_index--) + if (fd_index != 0) { + /* Windows implementation has only one fd per port_index. */ + return -1; + } + for (sp = s->head; sp && port_index != 0; sp = sp->next, --port_index) ; - if (port_index == 0 && sp) { + if (sp) { return _open_osfhandle(sp->socket->socket, 0); } else { return -1; @@ -485,25 +540,4 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, gpr_mu_unlock(&s->mu); } -int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) { - if (listener != NULL) { - grpc_tcp_listener *sp = listener; - return sp->port; - } else { - return 0; - } -} - -void grpc_tcp_listener_ref(grpc_tcp_listener *listener) { - grpc_tcp_listener *sp = listener; - gpr_ref(&sp->refs); -} - -void grpc_tcp_listener_unref(grpc_tcp_listener *listener) { - grpc_tcp_listener *sp = listener; - if (gpr_unref(&sp->refs)) { - gpr_free(listener); - } -} - #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c index d7fad338549..f4f3ff49218 100644 --- a/src/core/security/server_secure_chttp2.c +++ b/src/core/security/server_secure_chttp2.c @@ -126,8 +126,10 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep, state_unref(state); } -static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, - grpc_endpoint *tcp) { +static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp, + grpc_tcp_server *from_server, unsigned port_index, + unsigned fd_index) { + grpc_tcp_server_unref(NULL, from_server); grpc_server_secure_state *state = statep; state_ref(state); grpc_security_connector_do_handshake(exec_ctx, state->sc, tcp, @@ -144,8 +146,10 @@ static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep, static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, int success) { grpc_server_secure_state *state = statep; - state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg, - success); + if (state->destroy_callback != NULL) { + state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg, + success); + } grpc_security_connector_shutdown(exec_ctx, state->sc); state_unref(state); } @@ -161,8 +165,7 @@ static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep, state->destroy_callback = callback; tcp = state->tcp; gpr_mu_unlock(&state->mu); - grpc_closure_init(&state->destroy_closure, destroy_done, state); - grpc_tcp_server_destroy(exec_ctx, tcp, &state->destroy_closure); + grpc_tcp_server_unref(exec_ctx, tcp); } int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, @@ -199,18 +202,18 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, if (!resolved) { goto error; } - - tcp = grpc_tcp_server_create(); + state = gpr_malloc(sizeof(*state)); + memset(state, 0, sizeof(*state)); + grpc_closure_init(&state->destroy_closure, destroy_done, state); + tcp = grpc_tcp_server_create(&state->destroy_closure); if (!tcp) { goto error; } for (i = 0; i < resolved->naddrs; i++) { - grpc_tcp_listener *listener; - listener = grpc_tcp_server_add_port( + port_temp = grpc_tcp_server_add_port( tcp, (struct sockaddr *)&resolved->addrs[i].addr, resolved->addrs[i].len); - port_temp = grpc_tcp_listener_get_port(listener); if (port_temp > 0) { if (port_num == -1) { port_num = port_temp; @@ -232,8 +235,6 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, } grpc_resolved_addresses_destroy(resolved); - state = gpr_malloc(sizeof(*state)); - memset(state, 0, sizeof(*state)); state->server = server; state->tcp = tcp; state->sc = sc; @@ -258,7 +259,7 @@ error: grpc_resolved_addresses_destroy(resolved); } if (tcp) { - grpc_tcp_server_destroy(&exec_ctx, tcp, NULL); + grpc_tcp_server_unref(&exec_ctx, tcp); } if (state) { gpr_free(state); diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c index 5ce7c1955b8..f0b9f211b3d 100644 --- a/src/core/surface/server_chttp2.c +++ b/src/core/surface/server_chttp2.c @@ -53,7 +53,8 @@ static void setup_transport(grpc_exec_ctx *exec_ctx, void *server, } static void new_transport(grpc_exec_ctx *exec_ctx, void *server, - grpc_endpoint *tcp) { + grpc_endpoint *tcp, grpc_tcp_server *tcp_server, + unsigned port_index, unsigned fd_index) { /* * Beware that the call to grpc_create_chttp2_transport() has to happen before * grpc_tcp_server_destroy(). This is fine here, but similar code @@ -65,6 +66,7 @@ static void new_transport(grpc_exec_ctx *exec_ctx, void *server, exec_ctx, grpc_server_get_channel_args(server), tcp, 0); setup_transport(exec_ctx, server, transport); grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0); + grpc_tcp_server_unref(exec_ctx, tcp_server); } /* Server callback: start listening on our ports */ @@ -80,7 +82,8 @@ static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp, static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp, grpc_closure *destroy_done) { grpc_tcp_server *tcp = tcpp; - grpc_tcp_server_destroy(exec_ctx, tcp, destroy_done); + grpc_tcp_server_set_shutdown_complete(tcp, destroy_done); + grpc_tcp_server_unref(exec_ctx, tcp); } int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { @@ -100,15 +103,13 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { goto error; } - tcp = grpc_tcp_server_create(); + tcp = grpc_tcp_server_create(NULL); GPR_ASSERT(tcp); for (i = 0; i < resolved->naddrs; i++) { - grpc_tcp_listener *listener; - listener = grpc_tcp_server_add_port( + port_temp = grpc_tcp_server_add_port( tcp, (struct sockaddr *)&resolved->addrs[i].addr, resolved->addrs[i].len); - port_temp = grpc_tcp_listener_get_port(listener); if (port_temp > 0) { if (port_num == -1) { port_num = port_temp; @@ -139,7 +140,7 @@ error: grpc_resolved_addresses_destroy(resolved); } if (tcp) { - grpc_tcp_server_destroy(&exec_ctx, tcp, NULL); + grpc_tcp_server_unref(&exec_ctx, tcp); } port_num = 0; diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c index ceca56c833b..68832a4b4e5 100644 --- a/test/core/client_config/set_initial_connect_string_test.c +++ b/test/core/client_config/set_initial_connect_string_test.c @@ -78,8 +78,11 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { } } -static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) { +static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, + grpc_tcp_server *tcp_server, unsigned port_index, + unsigned fd_index) { test_tcp_server *server = arg; + grpc_tcp_server_unref(NULL, tcp_server); grpc_closure_init(&on_read, handle_read, NULL); gpr_slice_buffer_init(&state.incoming_buffer); gpr_slice_buffer_init(&state.temp_incoming_buffer); diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 530381e37fe..2791087b8a0 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -48,61 +48,70 @@ static grpc_pollset g_pollset; static int g_nconnects = 0; -static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) { +struct on_connect_result { + int server_fd; +}; + +static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, + grpc_tcp_server *tcp_server, unsigned port_index, + unsigned fd_index) { + struct on_connect_result *result = arg; grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + result->server_fd = grpc_tcp_server_get_fd(tcp_server, port_index, fd_index); g_nconnects++; grpc_pollset_kick(&g_pollset, NULL); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_tcp_server_unref(exec_ctx, tcp_server); } static void test_no_op(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_tcp_server *s = grpc_tcp_server_create(); - grpc_tcp_server_destroy(&exec_ctx, s, NULL); + grpc_tcp_server *s = grpc_tcp_server_create(NULL); + grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); } static void test_no_op_with_start(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_tcp_server *s = grpc_tcp_server_create(); + grpc_tcp_server *s = grpc_tcp_server_create(NULL); LOG_TEST("test_no_op_with_start"); grpc_tcp_server_start(&exec_ctx, s, NULL, 0, on_connect, NULL); - grpc_tcp_server_destroy(&exec_ctx, s, NULL); + grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); } static void test_no_op_with_port(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; struct sockaddr_in addr; - grpc_tcp_server *s = grpc_tcp_server_create(); + grpc_tcp_server *s = grpc_tcp_server_create(NULL); LOG_TEST("test_no_op_with_port"); memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; GPR_ASSERT( - grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr))); + grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)) > 0); - grpc_tcp_server_destroy(&exec_ctx, s, NULL); + grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); } static void test_no_op_with_port_and_start(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; struct sockaddr_in addr; - grpc_tcp_server *s = grpc_tcp_server_create(); + grpc_tcp_server *s = grpc_tcp_server_create(NULL); LOG_TEST("test_no_op_with_port_and_start"); memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; GPR_ASSERT( - grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr))); + grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)) > 0); grpc_tcp_server_start(&exec_ctx, s, NULL, 0, on_connect, NULL); - grpc_tcp_server_destroy(&exec_ctx, s, NULL); + grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); } @@ -111,32 +120,40 @@ static void test_connect(int n) { struct sockaddr_storage addr; socklen_t addr_len = sizeof(addr); int svrfd, clifd; - grpc_tcp_server *s = grpc_tcp_server_create(); + grpc_tcp_server *s = grpc_tcp_server_create(NULL); int nconnects_before; gpr_timespec deadline; grpc_pollset *pollsets[1]; int i; + struct on_connect_result result; LOG_TEST("test_connect"); gpr_log(GPR_INFO, "clients=%d", n); memset(&addr, 0, sizeof(addr)); addr.ss_family = AF_INET; - GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len)); - - svrfd = grpc_tcp_server_get_fd(s, 0); + GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len) > + 0); + GPR_ASSERT(grpc_tcp_server_fds_for_port(s, 2) == 0); + GPR_ASSERT(grpc_tcp_server_fds_for_port(s, 1) == 0); + GPR_ASSERT(grpc_tcp_server_fds_for_port(s, 0) == 1); + GPR_ASSERT(grpc_tcp_server_get_fd(s, 0, 1) < 0); + GPR_ASSERT(grpc_tcp_server_get_fd(s, 0, 2) < 0); + GPR_ASSERT(grpc_tcp_server_get_fd(s, 2, 0) < 0); + GPR_ASSERT(grpc_tcp_server_get_fd(s, 1, 0) < 0); + svrfd = grpc_tcp_server_get_fd(s, 0, 0); GPR_ASSERT(svrfd >= 0); GPR_ASSERT(getsockname(svrfd, (struct sockaddr *)&addr, &addr_len) == 0); GPR_ASSERT(addr_len <= sizeof(addr)); pollsets[0] = &g_pollset; - grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL); + grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, &result); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); for (i = 0; i < n; i++) { deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); - nconnects_before = g_nconnects; + result.server_fd = -1; clifd = socket(addr.ss_family, SOCK_STREAM, 0); GPR_ASSERT(clifd >= 0); gpr_log(GPR_DEBUG, "start connect"); @@ -156,11 +173,12 @@ static void test_connect(int n) { GPR_ASSERT(g_nconnects == nconnects_before + 1); close(clifd); + GPR_ASSERT(svrfd == result.server_fd); } gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); - grpc_tcp_server_destroy(&exec_ctx, s, NULL); + grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/test/core/util/reconnect_server.c b/test/core/util/reconnect_server.c index 28e521221b9..0db25703c4a 100644 --- a/test/core/util/reconnect_server.c +++ b/test/core/util/reconnect_server.c @@ -66,12 +66,15 @@ static void pretty_print_backoffs(reconnect_server *server) { } } -static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) { +static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, + grpc_tcp_server *tcp_server, unsigned port_index, + unsigned fd_index) { char *peer; char *last_colon; reconnect_server *server = (reconnect_server *)arg; gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); timestamp_list *new_tail; + grpc_tcp_server_unref(NULL, tcp_server); peer = grpc_endpoint_get_peer(tcp); grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 53b574d2856..cd7cd8db6af 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -58,7 +58,6 @@ void test_tcp_server_init(test_tcp_server *server, void test_tcp_server_start(test_tcp_server *server, int port) { struct sockaddr_in addr; - grpc_tcp_listener *listener; int port_added; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -66,9 +65,9 @@ void test_tcp_server_start(test_tcp_server *server, int port) { addr.sin_port = htons((uint16_t)port); memset(&addr.sin_addr, 0, sizeof(addr.sin_addr)); - server->tcp_server = grpc_tcp_server_create(); - listener = grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr)); - port_added = grpc_tcp_listener_get_port(listener); + server->tcp_server = grpc_tcp_server_create(NULL); + port_added = + grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr)); GPR_ASSERT(port_added == port); grpc_tcp_server_start(&exec_ctx, server->tcp_server, server->pollsets, 1, @@ -106,7 +105,9 @@ void test_tcp_server_destroy(test_tcp_server *server) { grpc_closure do_nothing_cb; grpc_closure_init(&server_shutdown_cb, on_server_destroyed, server); grpc_closure_init(&do_nothing_cb, do_nothing, NULL); - grpc_tcp_server_destroy(&exec_ctx, server->tcp_server, &server_shutdown_cb); + grpc_tcp_server_set_shutdown_complete(server->tcp_server, + &server_shutdown_cb); + grpc_tcp_server_unref(&exec_ctx, server->tcp_server); shutdown_deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(5, GPR_TIMESPAN)); while (!server->shutdown && From bb30d2591fc52c5bacc309c107077d92d5afc70a Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Tue, 12 Jan 2016 17:36:05 -0800 Subject: [PATCH 02/42] initial commit --- BUILD | 6 + Makefile | 37 ++ binding.gyp | 1 + build.yaml | 12 + gRPC.podspec | 5 +- include/grpc/census.h | 102 ++-- src/core/census/tag_set.c | 527 ++++++++++++++++++ src/core/census/tag_set.h | 57 ++ test/core/census/tag_set_test.c | 404 ++++++++++++++ tools/doxygen/Doxyfile.core.internal | 2 + tools/run_tests/sources_and_headers.json | 20 + tools/run_tests/tests.json | 19 + vsprojects/buildtests_c.sln | 27 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + .../test/tag_set_test/tag_set_test.vcxproj | 197 +++++++ .../tag_set_test/tag_set_test.vcxproj.filters | 21 + 19 files changed, 1408 insertions(+), 47 deletions(-) create mode 100644 src/core/census/tag_set.c create mode 100644 src/core/census/tag_set.h create mode 100644 test/core/census/tag_set_test.c create mode 100644 vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj create mode 100644 vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj.filters diff --git a/BUILD b/BUILD index 2b386fb814c..ae1a69073fc 100644 --- a/BUILD +++ b/BUILD @@ -268,6 +268,7 @@ cc_library( "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.h", "src/core/httpcli/httpcli_security_connector.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", @@ -417,6 +418,7 @@ cc_library( "src/core/census/context.c", "src/core/census/initialize.c", "src/core/census/operation.c", + "src/core/census/tag_set.c", "src/core/census/tracing.c", ], hdrs = [ @@ -559,6 +561,7 @@ cc_library( "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.h", "src/core/surface/init_unsecure.c", "src/core/census/grpc_context.c", "src/core/census/grpc_filter.c", @@ -688,6 +691,7 @@ cc_library( "src/core/census/context.c", "src/core/census/initialize.c", "src/core/census/operation.c", + "src/core/census/tag_set.c", "src/core/census/tracing.c", ], hdrs = [ @@ -1222,6 +1226,7 @@ objc_library( "src/core/census/context.c", "src/core/census/initialize.c", "src/core/census/operation.c", + "src/core/census/tag_set.c", "src/core/census/tracing.c", ], hdrs = [ @@ -1362,6 +1367,7 @@ objc_library( "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.h", ], includes = [ "include", diff --git a/Makefile b/Makefile index 449077b65c7..3d607c2174c 100644 --- a/Makefile +++ b/Makefile @@ -875,6 +875,7 @@ set_initial_connect_string_test: $(BINDIR)/$(CONFIG)/set_initial_connect_string_ sockaddr_resolver_test: $(BINDIR)/$(CONFIG)/sockaddr_resolver_test sockaddr_utils_test: $(BINDIR)/$(CONFIG)/sockaddr_utils_test socket_utils_test: $(BINDIR)/$(CONFIG)/socket_utils_test +tag_set_test: $(BINDIR)/$(CONFIG)/tag_set_test tcp_client_posix_test: $(BINDIR)/$(CONFIG)/tcp_client_posix_test tcp_posix_test: $(BINDIR)/$(CONFIG)/tcp_posix_test tcp_server_posix_test: $(BINDIR)/$(CONFIG)/tcp_server_posix_test @@ -1180,6 +1181,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/sockaddr_resolver_test \ $(BINDIR)/$(CONFIG)/sockaddr_utils_test \ $(BINDIR)/$(CONFIG)/socket_utils_test \ + $(BINDIR)/$(CONFIG)/tag_set_test \ $(BINDIR)/$(CONFIG)/tcp_client_posix_test \ $(BINDIR)/$(CONFIG)/tcp_posix_test \ $(BINDIR)/$(CONFIG)/tcp_server_posix_test \ @@ -1471,6 +1473,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/sockaddr_utils_test || ( echo test sockaddr_utils_test failed ; exit 1 ) $(E) "[RUN] Testing socket_utils_test" $(Q) $(BINDIR)/$(CONFIG)/socket_utils_test || ( echo test socket_utils_test failed ; exit 1 ) + $(E) "[RUN] Testing tag_set_test" + $(Q) $(BINDIR)/$(CONFIG)/tag_set_test || ( echo test tag_set_test failed ; exit 1 ) $(E) "[RUN] Testing tcp_client_posix_test" $(Q) $(BINDIR)/$(CONFIG)/tcp_client_posix_test || ( echo test tcp_client_posix_test failed ; exit 1 ) $(E) "[RUN] Testing tcp_posix_test" @@ -2446,6 +2450,7 @@ LIBGRPC_SRC = \ src/core/census/context.c \ src/core/census/initialize.c \ src/core/census/operation.c \ + src/core/census/tag_set.c \ src/core/census/tracing.c \ PUBLIC_HEADERS_C += \ @@ -2748,6 +2753,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/census/context.c \ src/core/census/initialize.c \ src/core/census/operation.c \ + src/core/census/tag_set.c \ src/core/census/tracing.c \ PUBLIC_HEADERS_C += \ @@ -8190,6 +8196,37 @@ endif endif +TAG_SET_TEST_SRC = \ + test/core/census/tag_set_test.c \ + +TAG_SET_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(TAG_SET_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/tag_set_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/tag_set_test: $(TAG_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(TAG_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/tag_set_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/census/tag_set_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +deps_tag_set_test: $(TAG_SET_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(TAG_SET_TEST_OBJS:.o=.dep) +endif +endif + + TCP_CLIENT_POSIX_TEST_SRC = \ test/core/iomgr/tcp_client_posix_test.c \ diff --git a/binding.gyp b/binding.gyp index 75e2f3c8de5..cd4265f0c34 100644 --- a/binding.gyp +++ b/binding.gyp @@ -301,6 +301,7 @@ 'src/core/census/context.c', 'src/core/census/initialize.c', 'src/core/census/operation.c', + 'src/core/census/tag_set.c', 'src/core/census/tracing.c', ], "conditions": [ diff --git a/build.yaml b/build.yaml index 6227b18b7d6..367ef98401f 100644 --- a/build.yaml +++ b/build.yaml @@ -16,10 +16,12 @@ filegroups: - src/core/census/aggregation.h - src/core/census/context.h - src/core/census/rpc_metric_id.h + - src/core/census/tag_set.h src: - src/core/census/context.c - src/core/census/initialize.c - src/core/census/operation.c + - src/core/census/tag_set.c - src/core/census/tracing.c - name: grpc++_base public_headers: @@ -1617,6 +1619,16 @@ targets: - mac - linux - posix +- name: tag_set_test + build: test + language: c + src: + - test/core/census/tag_set_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: tcp_client_posix_test build: test language: c diff --git a/gRPC.podspec b/gRPC.podspec index 80f26817d09..4acf2d7e1ba 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -272,6 +272,7 @@ Pod::Spec.new do |s| 'src/core/census/aggregation.h', 'src/core/census/context.h', 'src/core/census/rpc_metric_id.h', + 'src/core/census/tag_set.h', 'include/grpc/grpc_security.h', 'include/grpc/byte_buffer.h', 'include/grpc/byte_buffer_reader.h', @@ -428,6 +429,7 @@ Pod::Spec.new do |s| 'src/core/census/context.c', 'src/core/census/initialize.c', 'src/core/census/operation.c', + 'src/core/census/tag_set.c', 'src/core/census/tracing.c' ss.private_header_files = 'src/core/profiling/timers.h', @@ -569,7 +571,8 @@ Pod::Spec.new do |s| 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', 'src/core/census/context.h', - 'src/core/census/rpc_metric_id.h' + 'src/core/census/rpc_metric_id.h', + 'src/core/census/tag_set.h' ss.header_mappings_dir = '.' # This isn't officially supported in Cocoapods. We've asked for an alternative: diff --git a/include/grpc/census.h b/include/grpc/census.h index d0bc90420c0..b1276696944 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -324,60 +324,70 @@ int census_get_trace_record(census_trace_record *trace_record); /** End a scan previously started by census_trace_scan_start() */ void census_trace_scan_end(); -/* Max number of characters in tag key */ -#define CENSUS_MAX_TAG_KEY_LENGTH 20 -/* Max number of tag value characters */ -#define CENSUS_MAX_TAG_VALUE_LENGTH 50 - /* A Census tag set is a collection of key:value string pairs; these form the basis against which Census metrics will be recorded. Keys are unique within a tag set. All contexts have an associated tag set. */ typedef struct census_tag_set census_tag_set; -/* Returns a pointer to a newly created, empty tag set. If size_hint > 0, - indicates that the tag set is intended to hold approximately that number - of tags. */ -census_tag_set *census_tag_set_create(size_t size_hint); - -/* Add a new tag key/value to an existing tag set; if the tag key already exists - in the tag set, then its value is overwritten with the new one. Can also be - used to delete a tag, by specifying a NULL value. If key is NULL, returns - the number of tags in the tag set. - Return values: - -1: invalid length key or value - non-negative value: the number of tags in the tag set. */ -int census_tag_set_add(census_tag_set *tags, const char *key, - const char *value); - -/* Destroys a tag set. This function must be called to prevent memory leaks. - Once called, the tag set cannot be used again. */ -void census_tag_set_destroy(census_tag_set *tags); - -/* Get a contexts tag set. */ -census_tag_set *census_context_tag_set(census_context *context); - -/* A read-only representation of a tag for use by census clients. */ +/* A tag is a key:value pair. The key is a printable, nil-terminate + string. The value is a binary string, that may be printable. There are no + limits on the sizes of either keys or values, but code authors should + remember that systems may have inbuilt limits (e.g. for propagated tags, + the bytes on the wire) and that larger tags means more memory consumed and + time in processing. */ typedef struct { - size_t key_len; /* Number of bytes in tag key. */ - const char *key; /* A pointer to the tag key. May not be null-terminated. */ - size_t value_len; /* Number of bytes in tag value. */ - const char *value; /* Pointer to the tag value. May not be null-terminated. */ -} census_tag_const; + const char *key; + const char *value; + size_t value_len; + gpr_uint8 flags; +} census_tag; + +/* Tag flags. */ +#define CENSUS_TAG_PROPAGATE 1 /* Tag should be propagated over RPC */ +#define CENSUS_TAG_STATS 2 /* Tag will be used for statistics aggregation */ +#define CENSUS_TAG_BINARY 4 /* Tag value is not printable */ +#define CENSUS_TAG_RESERVED 8 /* Reserved for internal use. */ +/* Flag values 8,16,32,64,128 are reserved for future/internal use. Clients + should not use or rely on their values. */ + +#define CENSUS_TAG_IS_PROPAGATED(flags) (flags & CENSUS_TAG_PROPAGATE) +#define CENSUS_TAG_IS_STATS(flags) (flags & CENSUS_TAG_STATS) +#define CENSUS_TAG_IS_BINARY(flags) (flags & CENSUS_TAG_BINARY) + +#define CENSUS_MAX_TAG_KV_LEN 255 /* Maximum length of key/value in a tag. */ +#define CENSUS_MAX_TAGS 255 /* Maximum number of tags in a tag set. */ + +/* Create a new tag set, adding and removing tags from an existing tag set. + @param base Base tag set to build upon. Can be NULL. + @param tags A set of tags to be added/changed/deleted. Tags with keys that + are in 'tags', but not 'base', are added to the tag set. Keys that are in + both 'tags' and 'base' will have their value replaced. Tags with keys in + both, but with NULL or zero-length values, will be deleted from the + tag set. + @param ntags number of tags in 'tags' +*/ +census_tag_set *census_tag_set_create(const census_tag_set *base, + const census_tag *tags, int ntags); + +/* Destroy a tag set created by census_tag_set_create(). Once this function + has been called, the tag set cannot be reused. */ +void census_tag_set_destroy(census_tag_set *tags); -/* Used to iterate through a tag sets contents. */ -typedef struct census_tag_set_iterator census_tag_set_iterator; +/* Get the number of tags in the tag set. */ +int census_tag_set_ntags(const census_tag_set *tags); -/* Open a tag set for iteration. The tag set must not be modified while - iteration is ongoing. Returns an iterator for use in following functions. */ -census_tag_set_iterator *census_tag_set_open(census_tag_set *tags); +/* Get a tag by it's index in the tag set. Returns 0 if the index is invalid + (<0 or >= census_tag_set_ntags). There is no guarantee on tag ordering. */ +int census_tag_set_get_tag_by_index(const census_tag_set *tags, int index, + census_tag *tag); -/* Get the next tag in the tag set, by writing into the 'tag' argument. Returns - 1 if there is a "next" tag, 0 if there are no more tags. */ -int census_tag_set_next(census_tag_set_iterator *it, census_tag_const *tag); +/* Get a tag by its key. Returns 0 if the key is not present in the tag + set. */ +int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, + census_tag *tag); -/* Close an iterator opened by census_tag_set_open(). The iterator will be - invalidated, and should not be used once close is called. */ -void census_tag_set_close(census_tag_set_iterator *it); +/* Get a contexts tag set. */ +census_tag_set *census_context_tag_set(census_context *context); /* Core stats collection API's. The following concepts are used: * Aggregation: A collection of values. Census supports the following @@ -424,8 +434,8 @@ extern census_aggregation_ops census_agg_window; construction via census_define_view(). */ typedef struct { const census_aggregation_ops *ops; - const void * - create_arg; /* Argument to be used for aggregation initialization. */ + const void + *create_arg; /* Argument to be used for aggregation initialization. */ } census_aggregation; /** A census view type. Opaque. */ diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c new file mode 100644 index 00000000000..80c209031f4 --- /dev/null +++ b/src/core/census/tag_set.c @@ -0,0 +1,527 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "tag_set.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/core/support/murmur_hash.h" +#include "src/core/support/string.h" + +// Functions in this file support the public tag_set API, as well as +// encoding/decoding tag_sets as part of context transmission across +// RPC's. The overall requirements (in approximate priority order) for the +// tag_set representations: +// 1. Efficient conversion to/from wire format +// 2. Minimal bytes used on-wire +// 3. Efficient tag set creation +// 4. Efficient lookup of value for a key +// 5. Efficient lookup of value for an index (to support iteration) +// 6. Minimal memory footprint +// +// Notes on tradeoffs/decisions: +// * tag includes 1 byte length of key, as well as nil-terminating byte. These +// are to aid in efficient parsing and the ability to directly return key +// strings. This is more important than saving a single byte/tag on the wire. +// * The wire encoding uses only single byte values. This eliminates the need +// to handle endian-ness conversions. +// * Keep all tag information (keys/values/flags) in a single memory buffer, +// that can be directly copied to the wire. This makes iteration by index +// somewhat less efficient. +// * Binary tags are encoded seperately from non-binary tags. There are a +// primarily because non-binary tags are far more likely to be repeated +// across multiple RPC calls, so are more efficiently cached and +// compressed in any metadata schemes. +// * deleted/modified tags are kept in memory, just marked with a deleted +// flag. This enables faster processing TODO: benchmark this +// * all lengths etc. are restricted to one byte. This eliminates endian +// issues. + +// Structure representing a set of tags. Essentially a count of number of tags +// present, and pointer to a chunk of memory that contains the per-tag details. +struct tag_set { + int ntags; // number of tags. + int ntags_alloc; // ntags + number of deleted tags (total number of tags + // in all of kvm). + size_t kvm_size; // number of bytes allocated for key/value storage. + size_t kvm_used; // number of bytes of used key/value memory + char *kvm; // key/value memory. Consists of repeated entries of: + // Offset Size Description + // 0 1 Key length, including trailing 0. (K) + // 1 1 Value length. (V) + // 2 1 Flags + // 3 K Key bytes + // 3 + K V Value bytes + // + // We refer to the first 3 entries as the 'tag header'. +}; + +// Number of bytes in tag header. +#define TAG_HEADER_SIZE 3 // key length (1) + value length (1) + flags (1) +// Offsets to tag header entries. +#define KEY_LEN_OFFSET 0 +#define VALUE_LEN_OFFSET 1 +#define FLAG_OFFSET 2 + +// raw_tag represents the raw-storage form of a tag in the kvm of a tag_set. +struct raw_tag { + uint8_t key_len; + uint8_t value_len; + uint8_t flags; + char *key; + char *value; +}; + +// use reserved flag bit for indication of deleted tag. +#define CENSUS_TAG_DELETED CENSUS_TAG_RESERVED +#define CENSUS_TAG_IS_DELETED(flags) (flags & CENSUS_TAG_DELETED) + +// Primary (external) representation of a tag set. Composed of 3 underlying +// tag_set structs, one for each of the binary/printable propagated tags, and +// one for everything else. +struct census_tag_set { + struct tag_set propagated_tags; + struct tag_set propagated_binary_tags; + struct tag_set local_tags; +}; + +// Extract a raw tag given a pointer (raw) to the tag header. Allow for some +// extra bytes in the tag header (see encode/decode for usage: allows for +// future expansion of the tag header). +static char *decode_tag(struct raw_tag *tag, char *header, int offset) { + tag->key_len = (uint8_t)(*header++); + tag->value_len = (uint8_t)(*header++); + tag->flags = (uint8_t)(*header++); + header += offset; + tag->key = header; + header += tag->key_len; + tag->value = header; + return header + tag->value_len; +} + +// Make a copy (in 'to') of an existing tag_set. +static void tag_set_copy(struct tag_set *to, const struct tag_set *from) { + memcpy(to, from, sizeof(struct tag_set)); + to->kvm = gpr_malloc(to->kvm_size); + memcpy(to->kvm, from->kvm, to->kvm_used); +} + +// We may want to keep information about a deleted tag for a short time, +// in case we can reuse the space (same tag is reinserted). This structure +// is used for that purpose. +struct deleted_tag_info { + struct raw_tag raw; // raw tag information. + uint8_t *raw_flags_p; // pointer to original flags + struct tag_set *tags; // the tag set from which tag was deleted. +}; + +// Delete a tag from a tag set, if it exists. Returns true if the tag was +// originally present (and is now deleted), false if it wasn't. +static bool tag_set_delete_tag(struct tag_set *tags, + struct deleted_tag_info *dtag, const char *key, + size_t key_len) { + char *kvp = tags->kvm; + for (int i = 0; i < tags->ntags_alloc; i++) { + dtag->raw_flags_p = (uint8_t *)(kvp + FLAG_OFFSET); + kvp = decode_tag(&dtag->raw, kvp, 0); + if (CENSUS_TAG_IS_DELETED(dtag->raw.flags)) continue; + if ((key_len == dtag->raw.key_len) && + (memcmp(key, dtag->raw.key, key_len) == 0)) { + *(dtag->raw_flags_p) |= CENSUS_TAG_DELETED; + tags->ntags--; + return true; + } + } + return false; +} + +// Delete a tag from a tag set. If the tag is found in any of the underlying +// tag sets, *and* that tag set corresponds to the one in which the tag (if +// later inserted) would be placed, then fills in dtag, and returns true. +// Returns false otherwise. This information is later used to optimize the +// placement of the tag if the value space can be reused, effectively +// "undeleting" the tag. +static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, + size_t key_len, struct deleted_tag_info *dtag) { + // use the to-be-deleted tag flags as a hint to determine the order in which + // we delete from the underlying tag sets. + if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { + if (CENSUS_TAG_IS_BINARY(tag->flags)) { + if (tag_set_delete_tag(&tags->propagated_binary_tags, dtag, tag->key, + key_len)) { + dtag->tags = &tags->propagated_binary_tags; + return true; + } + if (tag_set_delete_tag(&tags->propagated_tags, dtag, tag->key, key_len)) + return false; + tag_set_delete_tag(&tags->local_tags, dtag, tag->key, key_len); + } else { + if (tag_set_delete_tag(&tags->propagated_tags, dtag, tag->key, key_len)) { + dtag->tags = &tags->propagated_tags; + return true; + } + if (tag_set_delete_tag(&tags->propagated_binary_tags, dtag, tag->key, + key_len)) + return false; + tag_set_delete_tag(&tags->local_tags, dtag, tag->key, key_len); + } + } else { + if (tag_set_delete_tag(&tags->local_tags, dtag, tag->key, key_len)) { + dtag->tags = &tags->local_tags; + return true; + } + if (tag_set_delete_tag(&tags->propagated_tags, dtag, tag->key, key_len)) + return false; + tag_set_delete_tag(&tags->propagated_binary_tags, dtag, tag->key, key_len); + } + return false; +} + +// Add a tag to a tag set. +static void tag_set_add_tag(struct tag_set *tags, const census_tag *tag, + size_t key_len) { + const size_t tag_size = key_len + tag->value_len + TAG_HEADER_SIZE; + // drop tag if too many. + if (tags->ntags == CENSUS_MAX_TAGS) { + return; + } + if (tags->kvm_used + tag_size > tags->kvm_size) { + tags->kvm_size += 2 * CENSUS_MAX_TAG_KV_LEN + TAG_HEADER_SIZE; + char *new_kvm = gpr_malloc(tags->kvm_size); + memcpy(new_kvm, tags->kvm, tags->kvm_used); + gpr_free(tags->kvm); + tags->kvm = new_kvm; + } + char *kvp = tags->kvm + tags->kvm_used; + *kvp++ = (char)key_len; + *kvp++ = (char)tag->value_len; + // ensure reserved flags are not used. + *kvp++ = (char)(tag->flags & (CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS | + CENSUS_TAG_BINARY)); + memcpy(kvp, tag->key, key_len); + kvp += key_len; + memcpy(kvp, tag->value, tag->value_len); + tags->kvm_used += tag_size; + tags->ntags++; + tags->ntags_alloc++; +} + +// Add a tag to a census_tag_set +static void cts_add_tag(census_tag_set *tags, const census_tag *tag, + size_t key_len) { + // first delete the tag if it is already present + struct deleted_tag_info dtag; + bool deleted_match = cts_delete_tag(tags, tag, key_len, &dtag); + if (tag->value != NULL && tag->value_len != 0) { + if (deleted_match && tag->value_len == dtag.raw.value_len) { + // if we have a close match for tag being added to one just deleted, + // only need to modify value and flags. + memcpy(dtag.raw.value, tag->value, tag->value_len); + *dtag.raw_flags_p = (tag->flags & (CENSUS_TAG_PROPAGATE | + CENSUS_TAG_STATS | CENSUS_TAG_BINARY)); + dtag.tags->ntags++; + } else { + if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { + if (CENSUS_TAG_IS_BINARY(tag->flags)) { + tag_set_add_tag(&tags->propagated_binary_tags, tag, key_len); + } else { + tag_set_add_tag(&tags->propagated_tags, tag, key_len); + } + } else { + tag_set_add_tag(&tags->local_tags, tag, key_len); + } + } + } +} + +census_tag_set *census_tag_set_create(const census_tag_set *base, + const census_tag *tags, int ntags) { + census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); + if (base == NULL) { + memset(new_ts, 0, sizeof(census_tag_set)); + } else { + tag_set_copy(&new_ts->propagated_tags, &base->propagated_tags); + tag_set_copy(&new_ts->propagated_binary_tags, + &base->propagated_binary_tags); + tag_set_copy(&new_ts->local_tags, &base->local_tags); + } + for (int i = 0; i < ntags; i++) { + const census_tag *tag = &tags[i]; + size_t key_len = strlen(tag->key) + 1; + // ignore the tag if it is too long/short. + if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN && + tag->value_len <= CENSUS_MAX_TAG_KV_LEN) { + cts_add_tag(new_ts, tag, key_len); + } + } + return new_ts; +} + +void census_tag_set_destroy(census_tag_set *tags) { + gpr_free(tags->propagated_tags.kvm); + gpr_free(tags->propagated_binary_tags.kvm); + gpr_free(tags->local_tags.kvm); + gpr_free(tags); +} + +int census_tag_set_ntags(const census_tag_set *tags) { + return tags->propagated_tags.ntags + tags->propagated_binary_tags.ntags + + tags->local_tags.ntags; +} + +// Get the nth tag in a tag set. The caller must validate that index is +// in range. +static void tag_set_get_tag_by_index(const struct tag_set *tags, int index, + census_tag *tag) { + GPR_ASSERT(index < tags->ntags); + char *kvp = tags->kvm; + for (;;) { + struct raw_tag raw; + kvp = decode_tag(&raw, kvp, 0); + if (CENSUS_TAG_IS_DELETED(raw.flags)) { + continue; + } else if (index == 0) { + tag->key = raw.key; + tag->value = raw.value; + tag->value_len = raw.value_len; + tag->flags = raw.flags; + return; + } + index--; + } + // NOT REACHED +} + +int census_tag_set_get_tag_by_index(const census_tag_set *tags, int index, + census_tag *tag) { + if (index < 0) return 0; + if (index < tags->propagated_tags.ntags) { + tag_set_get_tag_by_index(&tags->propagated_tags, index, tag); + return 1; + } + index -= tags->propagated_tags.ntags; + if (index < tags->propagated_binary_tags.ntags) { + tag_set_get_tag_by_index(&tags->propagated_binary_tags, index, tag); + return 1; + } + index -= tags->propagated_binary_tags.ntags; + if (index < tags->local_tags.ntags) { + tag_set_get_tag_by_index(&tags->local_tags, index, tag); + return 1; + } + return 0; +} + +// Find a tag in a tag_set by key. Return true if found, false otherwise. +static bool tag_set_get_tag_by_key(const struct tag_set *tags, const char *key, + size_t key_len, census_tag *tag) { + char *kvp = tags->kvm; + for (int i = 0; i < tags->ntags; i++) { + struct raw_tag raw; + do { + kvp = decode_tag(&raw, kvp, 0); + } while (CENSUS_TAG_IS_DELETED(raw.flags)); + if (key_len == raw.key_len && memcmp(raw.key, key, key_len) == 0) { + tag->key = raw.key; + tag->value = raw.value; + tag->value_len = raw.value_len; + tag->flags = raw.flags; + return true; + } + } + return false; +} + +int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, + census_tag *tag) { + size_t key_len = strlen(key) + 1; + if (key_len == 1) { + return 0; + } + if (tag_set_get_tag_by_key(&tags->propagated_tags, key, key_len, tag) || + tag_set_get_tag_by_key(&tags->propagated_binary_tags, key, key_len, + tag) || + tag_set_get_tag_by_key(&tags->local_tags, key, key_len, tag)) { + return 1; + } + return 0; +} + +// tag_set encoding and decoding functions. +// +// Wire format for tag sets on the wire: +// +// First, a tag set header: +// +// offset bytes description +// 0 1 version number +// 1 1 number of bytes in this header. This allows for future +// expansion. +// 2 1 number of bytes in each tag header. +// 3 1 ntags value from tag set. +// 4 1 ntags_alloc value from tag set. +// +// This is followed by the key/value memory from struct tag_set. + +#define ENCODED_VERSION 0 // Version number +#define ENCODED_HEADER_SIZE 5 // size of tag set header + +// Pack a tag set into as few bytes as possible (eliding deleted tags). Assumes +// header is already generated. +static size_t tag_set_encode_packed(const struct tag_set *tags, char *buffer, + size_t buf_size) { + size_t encoded_size = 0; + char *kvp = tags->kvm; + for (int i = 0; i < tags->ntags_alloc; i++) { + struct raw_tag raw; + char *base = kvp; + kvp = decode_tag(&raw, kvp, 0); + size_t tag_size = + TAG_HEADER_SIZE + (size_t)raw.key_len + (size_t)raw.value_len; + if (!(CENSUS_TAG_IS_DELETED(raw.flags))) { + if (tag_size > buf_size) { + return 0; + } + memcpy(buffer, base, tag_size); + buffer += tag_size; + encoded_size += tag_size; + buf_size -= tag_size; + } + } + return encoded_size; +} + +// Encode a tag set. Returns 0 if buffer is too small. +static size_t tag_set_encode(const struct tag_set *tags, char *buffer, + size_t buf_size) { + if (buf_size < ENCODED_HEADER_SIZE) { + return 0; + } + buf_size -= ENCODED_HEADER_SIZE; + *buffer++ = (char)ENCODED_VERSION; + *buffer++ = (char)ENCODED_HEADER_SIZE; + *buffer++ = (char)TAG_HEADER_SIZE; + *buffer++ = (char)tags->ntags; + if (tags->ntags == 0) { + *buffer = (char)tags->ntags; + return ENCODED_HEADER_SIZE; + } + if (buf_size < tags->kvm_used || tags->ntags_alloc > CENSUS_MAX_TAGS) { + *buffer++ = (char)tags->ntags; + size_t enc_size = tag_set_encode_packed(tags, buffer, buf_size); + if (enc_size == 0) { + return 0; + } + return ENCODED_HEADER_SIZE + enc_size; + } + *buffer++ = (char)tags->ntags_alloc; + memcpy(buffer, tags->kvm, tags->kvm_used); + return ENCODED_HEADER_SIZE + tags->kvm_used; +} + +size_t census_tag_set_encode_propagated(const census_tag_set *tags, + char *buffer, size_t buf_size) { + return tag_set_encode(&tags->propagated_tags, buffer, buf_size); +} + +size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, + char *buffer, size_t buf_size) { + return tag_set_encode(&tags->propagated_binary_tags, buffer, buf_size); +} + +// Decode a tag set. +static void tag_set_decode(struct tag_set *tags, const char *buffer, + size_t size) { + uint8_t version = (uint8_t)(*buffer++); + uint8_t header_size = (uint8_t)(*buffer++); + uint8_t tag_header_size = (uint8_t)(*buffer++); + tags->ntags = (int)(*buffer++); + if (tags->ntags == 0) { + tags->ntags_alloc = 0; + tags->kvm_size = 0; + tags->kvm_used = 0; + tags->kvm = NULL; + return; + } + tags->ntags_alloc = (uint8_t)(*buffer++); + if (header_size != ENCODED_HEADER_SIZE) { + GPR_ASSERT(version != ENCODED_VERSION); + GPR_ASSERT(ENCODED_HEADER_SIZE < header_size); + buffer += (header_size - ENCODED_HEADER_SIZE); + } + tags->kvm_used = size - header_size; + tags->kvm_size = tags->kvm_used + CENSUS_MAX_TAG_KV_LEN; + tags->kvm = gpr_malloc(tags->kvm_size); + if (tag_header_size != TAG_HEADER_SIZE) { + // something new in the tag information. I don't understand it, so + // don't copy it over. + GPR_ASSERT(version != ENCODED_VERSION); + GPR_ASSERT(tag_header_size > TAG_HEADER_SIZE); + char *kvp = tags->kvm; + for (int i = 0; i < tags->ntags_alloc; i++) { + memcpy(kvp, buffer, TAG_HEADER_SIZE); + kvp += header_size; + struct raw_tag raw; + buffer = + decode_tag(&raw, (char *)buffer, tag_header_size - TAG_HEADER_SIZE); + memcpy(kvp, raw.key, (size_t)raw.key_len + raw.value_len); + kvp += raw.key_len + raw.value_len; + } + } else { + memcpy(tags->kvm, buffer, tags->kvm_used); + } +} + +census_tag_set *census_tag_set_decode(const char *buffer, size_t size, + const char *bin_buffer, size_t bin_size) { + census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); + memset(&new_ts->local_tags, 0, sizeof(struct tag_set)); + if (buffer == NULL) { + memset(&new_ts->propagated_tags, 0, sizeof(struct tag_set)); + } else { + tag_set_decode(&new_ts->propagated_tags, buffer, size); + } + if (bin_buffer == NULL) { + memset(&new_ts->propagated_binary_tags, 0, sizeof(struct tag_set)); + } else { + tag_set_decode(&new_ts->propagated_binary_tags, bin_buffer, bin_size); + } + // TODO(aveitch): check that BINARY flag is correct for each type. + return new_ts; +} diff --git a/src/core/census/tag_set.h b/src/core/census/tag_set.h new file mode 100644 index 00000000000..9eec0ad4384 --- /dev/null +++ b/src/core/census/tag_set.h @@ -0,0 +1,57 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CENSUS_TAG_SET_H +#define GRPC_INTERNAL_CORE_CENSUS_TAG_SET_H + +#include +#include +#include + +/* Encode to-be-propagated tags from a tag set into a memory buffer. The total + number of bytes used in the buffer is returned. If the buffer is too small + to contain the encoded tag set, then 0 is returned. */ +size_t census_tag_set_encode_propagated(const census_tag_set *tags, + char *buffer, size_t buf_size); + +/* Encode to-be-propagated binary tags from a tag set into a memory + buffer. The total number of bytes used in the buffer is returned. If the + buffer is too small to contain the encoded tag set, then 0 is returned. */ +size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, + char *buffer, size_t buf_size); + +/* Decode tag set buffers encoded with census_tag_set_encode_*(). */ +census_tag_set *census_tag_set_decode(const char *buffer, size_t size, + const char *bin_buffer, size_t bin_size); + +#endif /* GRPC_INTERNAL_CORE_CENSUS_TAG_SET_H */ diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c new file mode 100644 index 00000000000..d6a7e453346 --- /dev/null +++ b/test/core/census/tag_set_test.c @@ -0,0 +1,404 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Test census_tag_set functions, including encoding/decoding + +#include "src/core/census/tag_set.h" +#include +#include +#include +#include +#include +#include +#include "test/core/util/test_config.h" + +static uint8_t one_byte_val = 7; +static uint32_t four_byte_val = 0x12345678; +static uint64_t eight_byte_val = 0x1234567890abcdef; + +// A set of tags Used to create a basic tag_set for testing. Each tag has a +// unique set of flags. Note that replace_add_delete_test() relies on specific +// offsets into this array - if you add or delete entries, you will also need +// to change the test. +#define BASIC_TAG_COUNT 8 +static census_tag basic_tags[BASIC_TAG_COUNT] = { + /* 0 */ {"key0", "printable", 10, 0}, + /* 1 */ {"k1", "a", 2, CENSUS_TAG_PROPAGATE}, + /* 2 */ {"k2", "longer printable string", 24, CENSUS_TAG_STATS}, + /* 3 */ {"key_three", (char *)&one_byte_val, 1, CENSUS_TAG_BINARY}, + /* 4 */ {"really_long_key_4", "random", 7, + CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS}, + /* 5 */ {"k5", (char *)&four_byte_val, 4, + CENSUS_TAG_PROPAGATE | CENSUS_TAG_BINARY}, + /* 6 */ {"k6", (char *)&eight_byte_val, 8, + CENSUS_TAG_STATS | CENSUS_TAG_BINARY}, + /* 7 */ {"k7", (char *)&four_byte_val, 4, + CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS | CENSUS_TAG_BINARY}}; + +// Set of tags used to modify the basic tag_set. Note that +// replace_add_delete_test() relies on specific offsets into this array - if +// you add or delete entries, you will also need to change the test. Other +// tests that rely on specific instances have XXX_XXX_OFFSET definitions (also +// change the defines below if you add/delete entires). +#define MODIFY_TAG_COUNT 10 +static census_tag modify_tags[MODIFY_TAG_COUNT] = { +#define REPLACE_VALUE_OFFSET 0 + /* 0 */ {"key0", "replace printable", 18, 0}, // replaces tag value only +#define ADD_TAG_OFFSET 1 + /* 1 */ {"new_key", "xyzzy", 6, CENSUS_TAG_STATS}, // new tag +#define DELETE_TAG_OFFSET 2 + /* 2 */ {"k5", NULL, 5, + 0}, // should delete tag, despite bogus value length + /* 3 */ {"k6", "foo", 0, 0}, // should delete tag, despite bogus value + /* 4 */ {"k6", "foo", 0, 0}, // try deleting already-deleted tag + /* 5 */ {"non-existent", NULL, 0, 0}, // another non-existent tag +#define REPLACE_FLAG_OFFSET 6 + /* 6 */ {"k1", "a", 2, 0}, // change flags only + /* 7 */ {"k7", "bar", 4, CENSUS_TAG_STATS}, // change flags and value + /* 8 */ {"k2", (char *)&eight_byte_val, 8, + CENSUS_TAG_BINARY | CENSUS_TAG_PROPAGATE}, // more flags change + // non-binary -> binary + /* 9 */ {"k6", "bar", 4, + 0} // add back tag, with different value, but same length +}; + +// Utility function to compare tags. Returns true if all fields match. +static bool compare_tag(const census_tag *t1, const census_tag *t2) { + return (strcmp(t1->key, t2->key) == 0 && t1->value_len == t2->value_len && + memcmp(t1->value, t2->value, t1->value_len) == 0 && + t1->flags == t2->flags); +} + +// Utility function to validate a tag exists in tag set. +static bool validate_tag(const census_tag_set *cts, const census_tag *tag) { + census_tag tag2; + if (census_tag_set_get_tag_by_key(cts, tag->key, &tag2) != 1) return false; + return compare_tag(tag, &tag2); +} + +// Create an empty tag_set. +static void empty_test(void) { + struct census_tag_set *cts = census_tag_set_create(NULL, NULL, 0); + GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_destroy(cts); +} + +// Create basic tag set, and test that retreiving tag by index works. +static void basic_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + for (int i = 0; i < census_tag_set_ntags(cts); i++) { + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_index(cts, i, &tag) == 1); + // can't rely on tag return order: make sure it matches exactly one. + int matches = 0; + for (int j = 0; j < BASIC_TAG_COUNT; j++) { + if (compare_tag(&tag, &basic_tags[j])) matches++; + } + GPR_ASSERT(matches == 1); + } + census_tag_set_destroy(cts); +} + +// Try census_tag_set_get_tag_by_index() with bad indices. +static void bad_index_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_index(cts, -1, &tag) == 0); + GPR_ASSERT(census_tag_set_get_tag_by_index(cts, BASIC_TAG_COUNT, &tag) == 0); + GPR_ASSERT(census_tag_set_get_tag_by_index(cts, BASIC_TAG_COUNT + 1, &tag) == + 0); + census_tag_set_destroy(cts); +} + +// Test that census_tag_set_get_tag_by_key(). +static void lookup_by_key_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag tag; + for (int i = 0; i < census_tag_set_ntags(cts); i++) { + GPR_ASSERT(census_tag_set_get_tag_by_key(cts, basic_tags[i].key, &tag) == + 1); + GPR_ASSERT(compare_tag(&tag, &basic_tags[i])); + } + // non-existent keys + GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "key", &tag) == 0); + GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "key01", &tag) == 0); + GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "k9", &tag) == 0); + GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "random", &tag) == 0); + GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "", &tag) == 0); + census_tag_set_destroy(cts); +} + +// Try creating tag set with invalid entries. +static void invalid_test(void) { + char key[300]; + memset(key, 'k', 299); + key[299] = 0; + char value[300]; + memset(value, 'v', 300); + census_tag tag = {key, value, 3, CENSUS_TAG_BINARY}; + // long keys, short value. Key lengths (including terminator) should be + // <= 255 (CENSUS_MAX_TAG_KV_LEN) + GPR_ASSERT(strlen(key) == 299); + struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_destroy(cts); + key[CENSUS_MAX_TAG_KV_LEN] = 0; + GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN); + cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_destroy(cts); + key[CENSUS_MAX_TAG_KV_LEN - 1] = 0; + GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN - 1); + cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 1); + census_tag_set_destroy(cts); + // now try with long values + tag.value_len = 300; + cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_destroy(cts); + tag.value_len = CENSUS_MAX_TAG_KV_LEN + 1; + cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_destroy(cts); + tag.value_len = CENSUS_MAX_TAG_KV_LEN; + cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 1); + census_tag_set_destroy(cts); + // 0 length key. + key[0] = 0; + cts = census_tag_set_create(NULL, &tag, 1); + GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_destroy(cts); +} + +// Make a copy of a tag set +static void copy_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0); + GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); + for (int i = 0; i < census_tag_set_ntags(cts2); i++) { + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) == + 1); + GPR_ASSERT(compare_tag(&tag, &basic_tags[i])); + } + census_tag_set_destroy(cts); + census_tag_set_destroy(cts2); +} + +// replace a single tag value +static void replace_value_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = + census_tag_set_create(cts, modify_tags + REPLACE_VALUE_OFFSET, 1); + GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_key( + cts2, modify_tags[REPLACE_VALUE_OFFSET].key, &tag) == 1); + GPR_ASSERT(compare_tag(&tag, &modify_tags[REPLACE_VALUE_OFFSET])); + census_tag_set_destroy(cts); + census_tag_set_destroy(cts2); +} + +// replace a single tags flags +static void replace_flags_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = + census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1); + GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_key( + cts2, modify_tags[REPLACE_FLAG_OFFSET].key, &tag) == 1); + GPR_ASSERT(compare_tag(&tag, &modify_tags[REPLACE_FLAG_OFFSET])); + census_tag_set_destroy(cts); + census_tag_set_destroy(cts2); +} + +// delete a single tag. +static void delete_tag_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = + census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1); + GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT - 1); + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_key( + cts2, modify_tags[DELETE_TAG_OFFSET].key, &tag) == 0); + census_tag_set_destroy(cts); + census_tag_set_destroy(cts2); +} + +// add a single new tag. +static void add_tag_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = + census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1); + GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT + 1); + census_tag tag; + GPR_ASSERT(census_tag_set_get_tag_by_key( + cts2, modify_tags[ADD_TAG_OFFSET].key, &tag) == 1); + GPR_ASSERT(compare_tag(&tag, &modify_tags[ADD_TAG_OFFSET])); + census_tag_set_destroy(cts); + census_tag_set_destroy(cts2); +} + +// test many changes at once. +static void replace_add_delete_test(void) { + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = + census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts2) == 8); + // validate tag set contents. Use specific indices into the two arrays + // holding tag values. + GPR_ASSERT(validate_tag(cts2, &basic_tags[3])); + GPR_ASSERT(validate_tag(cts2, &basic_tags[4])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[0])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[1])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[6])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[7])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[8])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[9])); + GPR_ASSERT(!validate_tag(cts2, &basic_tags[0])); + GPR_ASSERT(!validate_tag(cts2, &basic_tags[1])); + GPR_ASSERT(!validate_tag(cts2, &basic_tags[2])); + GPR_ASSERT(!validate_tag(cts2, &basic_tags[5])); + GPR_ASSERT(!validate_tag(cts2, &basic_tags[6])); + GPR_ASSERT(!validate_tag(cts2, &basic_tags[7])); + census_tag_set_destroy(cts); + census_tag_set_destroy(cts2); +} + +// Use the basic tag set to test encode/decode. +static void simple_encode_decode_test(void) { + char buf1[1000]; + char buf2[1000]; + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_encode_propagated(cts, buf1, 1) == 0); + size_t b1 = census_tag_set_encode_propagated(cts, buf1, 1000); + GPR_ASSERT(b1 != 0); + GPR_ASSERT(census_tag_set_encode_propagated_binary(cts, buf2, 1) == 0); + size_t b2 = census_tag_set_encode_propagated_binary(cts, buf2, 1000); + GPR_ASSERT(b2 != 0); + census_tag_set *cts2 = census_tag_set_decode(buf1, b1, buf2, b2); + GPR_ASSERT(cts2 != NULL); + GPR_ASSERT(census_tag_set_ntags(cts2) == 4); + for (int i = 0; i < census_tag_set_ntags(cts); i++) { + census_tag tag; + if (CENSUS_TAG_IS_PROPAGATED(basic_tags[i].flags)) { + GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) == + 1); + GPR_ASSERT(compare_tag(&tag, &basic_tags[i])); + } else { + GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) == + 0); + } + } + census_tag_set_destroy(cts2); + census_tag_set_destroy(cts); +} + +// Use more complex/modified tag set to test encode/decode. +static void complex_encode_decode_test(void) { + char buf1[500]; + char buf2[500]; + struct census_tag_set *cts = + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + struct census_tag_set *cts2 = + census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT); + GPR_ASSERT(census_tag_set_ntags(cts2) == 8); + + size_t b1 = census_tag_set_encode_propagated(cts2, buf1, 500); + GPR_ASSERT(b1 != 0); + size_t b2 = census_tag_set_encode_propagated_binary(cts2, buf2, 500); + GPR_ASSERT(b2 != 0); + census_tag_set *cts3 = census_tag_set_decode(buf1, b1, buf2, b2); + GPR_ASSERT(cts3 != NULL); + GPR_ASSERT(census_tag_set_ntags(cts3) == 2); + GPR_ASSERT(validate_tag(cts3, &basic_tags[4])); + GPR_ASSERT(validate_tag(cts3, &modify_tags[8])); + // Now force tag set to be in smaller space + census_tag_set_destroy(cts3); + size_t nb1 = census_tag_set_encode_propagated(cts2, buf1, b1 - 1); + GPR_ASSERT(nb1 != 0); + GPR_ASSERT(nb1 < b1); + size_t nb2 = census_tag_set_encode_propagated_binary(cts2, buf2, b2 - 1); + GPR_ASSERT(nb2 != 0); + GPR_ASSERT(nb2 < b2); + cts3 = census_tag_set_decode(buf1, nb1, buf2, nb2); + GPR_ASSERT(cts3 != NULL); + GPR_ASSERT(census_tag_set_ntags(cts3) == 2); + GPR_ASSERT(validate_tag(cts3, &basic_tags[4])); + GPR_ASSERT(validate_tag(cts3, &modify_tags[8])); + census_tag_set_destroy(cts3); + census_tag_set_destroy(cts2); + census_tag_set_destroy(cts); +} + +int main(int argc, char *argv[]) { + grpc_test_init(argc, argv); + empty_test(); + basic_test(); + bad_index_test(); + lookup_by_key_test(); + invalid_test(); + copy_test(); + replace_value_test(); + replace_flags_test(); + delete_tag_test(); + add_tag_test(); + replace_add_delete_test(); + simple_encode_decode_test(); + complex_encode_decode_test(); + return 0; +} diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index aef5bec86b0..ac57dd03b7e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -897,6 +897,7 @@ src/core/transport/transport_impl.h \ src/core/census/aggregation.h \ src/core/census/context.h \ src/core/census/rpc_metric_id.h \ +src/core/census/tag_set.h \ src/core/httpcli/httpcli_security_connector.c \ src/core/security/base64.c \ src/core/security/client_auth_filter.c \ @@ -1046,6 +1047,7 @@ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ src/core/census/operation.c \ +src/core/census/tag_set.c \ src/core/census/tracing.c \ include/grpc/support/alloc.h \ include/grpc/support/atm.h \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index a1beafd41e1..529a4173ead 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1089,6 +1089,20 @@ "test/core/iomgr/socket_utils_test.c" ] }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "tag_set_test", + "src": [ + "test/core/census/tag_set_test.c" + ] + }, { "deps": [ "gpr", @@ -2845,6 +2859,7 @@ "src/core/census/context.h", "src/core/census/grpc_filter.h", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", "src/core/channel/client_channel.h", @@ -2991,6 +3006,8 @@ "src/core/census/initialize.c", "src/core/census/operation.c", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.c", + "src/core/census/tag_set.h", "src/core/census/tracing.c", "src/core/channel/channel_args.c", "src/core/channel/channel_args.h", @@ -3357,6 +3374,7 @@ "src/core/census/context.h", "src/core/census/grpc_filter.h", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", "src/core/channel/client_channel.h", @@ -3488,6 +3506,8 @@ "src/core/census/initialize.c", "src/core/census/operation.c", "src/core/census/rpc_metric_id.h", + "src/core/census/tag_set.c", + "src/core/census/tag_set.h", "src/core/census/tracing.c", "src/core/channel/channel_args.c", "src/core/channel/channel_args.h", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 0b5f847af18..a8398eb8606 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1328,6 +1328,25 @@ "posix" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "tag_set_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 06e882ee95b..592b3984b6a 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -848,6 +848,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sockaddr_utils_test", "vcxp {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tag_set_test", "vcxproj\test\tag_set_test\tag_set_test.vcxproj", "{430F8F07-6AAD-0150-B35B-DB9E2E21941A}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "time_averaged_stats_test", "vcxproj\test\time_averaged_stats_test\time_averaged_stats_test.vcxproj", "{D1EB2A9B-8508-62D7-8FC4-11A11B1CBFD3}" ProjectSection(myProperties) = preProject lib = "False" @@ -2624,6 +2635,22 @@ Global {529771F0-10B0-9B1A-1E7E-8A8E01870348}.Release-DLL|Win32.Build.0 = Release|Win32 {529771F0-10B0-9B1A-1E7E-8A8E01870348}.Release-DLL|x64.ActiveCfg = Release|x64 {529771F0-10B0-9B1A-1E7E-8A8E01870348}.Release-DLL|x64.Build.0 = Release|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug|Win32.ActiveCfg = Debug|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug|x64.ActiveCfg = Debug|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release|Win32.ActiveCfg = Release|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release|x64.ActiveCfg = Release|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug|Win32.Build.0 = Debug|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug|x64.Build.0 = Debug|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release|Win32.Build.0 = Release|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release|x64.Build.0 = Release|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Debug-DLL|x64.Build.0 = Debug|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release-DLL|Win32.Build.0 = Release|Win32 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release-DLL|x64.ActiveCfg = Release|x64 + {430F8F07-6AAD-0150-B35B-DB9E2E21941A}.Release-DLL|x64.Build.0 = Release|x64 {D1EB2A9B-8508-62D7-8FC4-11A11B1CBFD3}.Debug|Win32.ActiveCfg = Debug|Win32 {D1EB2A9B-8508-62D7-8FC4-11A11B1CBFD3}.Debug|x64.ActiveCfg = Debug|x64 {D1EB2A9B-8508-62D7-8FC4-11A11B1CBFD3}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 9d646153e28..037b8ff2224 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -404,6 +404,7 @@ + @@ -704,6 +705,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 055256a7e81..0ddbd5e46d5 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -448,6 +448,9 @@ src\core\census + + src\core\census + src\core\census @@ -866,6 +869,9 @@ src\core\census + + src\core\census + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index b2965212bb4..a3532e5d05e 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -382,6 +382,7 @@ + @@ -642,6 +643,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index e30ca5f685d..f63e3012e75 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -388,6 +388,9 @@ src\core\census + + src\core\census + src\core\census @@ -761,6 +764,9 @@ src\core\census + + src\core\census + diff --git a/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj b/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj new file mode 100644 index 00000000000..38949026726 --- /dev/null +++ b/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj @@ -0,0 +1,197 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {430F8F07-6AAD-0150-B35B-DB9E2E21941A} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + tag_set_test + static + Debug + Debug + + + tag_set_test + static + Release + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj.filters b/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj.filters new file mode 100644 index 00000000000..6b315322519 --- /dev/null +++ b/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\census + + + + + + {500aa440-5924-8047-996a-4c5096d1ef96} + + + {a3bf80f0-5b13-f623-277b-05f0231dd933} + + + {b6ed1b86-7795-4da9-a169-9eccf836852c} + + + + From 5d81d15260a2f355580bc1ede89c69b456c71f9f Mon Sep 17 00:00:00 2001 From: Dan Born Date: Tue, 12 Jan 2016 20:29:29 -0800 Subject: [PATCH 03/42] Code review follow-up. --- src/core/iomgr/tcp_server.h | 26 +++++++++++-------- src/core/iomgr/tcp_server_posix.c | 23 +++++++--------- src/core/iomgr/tcp_server_windows.c | 22 +++++++--------- src/core/security/server_secure_chttp2.c | 4 +-- src/core/surface/server_chttp2.c | 7 +++-- .../set_initial_connect_string_test.c | 4 +-- test/core/iomgr/tcp_server_posix_test.c | 23 ++++++++-------- test/core/util/reconnect_server.c | 4 +-- test/core/util/test_tcp_server.c | 21 +++++++-------- test/core/util/test_tcp_server.h | 1 + tools/run_tests/run_tests.py | 8 +++--- 11 files changed, 66 insertions(+), 77 deletions(-) diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h index 92e504d20be..e4fd0999a9f 100644 --- a/src/core/iomgr/tcp_server.h +++ b/src/core/iomgr/tcp_server.h @@ -39,12 +39,20 @@ /* Forward decl of grpc_tcp_server */ typedef struct grpc_tcp_server grpc_tcp_server; -/* Called for newly connected TCP connections. Callee owns a ref on - from_server. */ +typedef struct grpc_tcp_server_acceptor grpc_tcp_server_acceptor; +struct grpc_tcp_server_acceptor { + /* grpc_tcp_server_cb functions share a ref on from_server that is valid + until the function returns. */ + grpc_tcp_server *from_server; + /* Indices that may be passed to grpc_tcp_server_port_fd(). */ + unsigned port_index; + unsigned fd_index; +}; + +/* Called for newly connected TCP connections. */ typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *ep, - grpc_tcp_server *from_server, - unsigned port_index, unsigned fd_index); + grpc_tcp_server_acceptor *acceptor); /* Create a server, initially not bound to any ports. The caller owns one ref. If shutdown_complete is not NULL, it will be used by @@ -70,22 +78,18 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, /* Number of fds at the given port_index, or 0 if port_index is out of bounds. */ -unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, unsigned port_index); +unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s, unsigned port_index); /* Returns the file descriptor of the Mth (fd_index) listening socket of the Nth (port_index) call to add_port() on this server, or -1 if the indices are out of bounds. The file descriptor remains owned by the server, and will be cleaned up when grpc_tcp_server_destroy is called. */ -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index, - unsigned fd_index); +int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index, + unsigned fd_index); /* Ref s and return s. */ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s); -/* Set or reset the shutdown_complete closure. shutdown_complete may be NULL. */ -void grpc_tcp_server_set_shutdown_complete(grpc_tcp_server *s, - grpc_closure *shutdown_complete); - /* If the recount drops to zero, delete s, and call (exec_ctx==NULL) or enqueue a call (exec_ctx!=NULL) to shutdown_complete. */ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s); diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index f1119f689ad..3e0c5be3483 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -212,8 +212,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { } } -static void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, - grpc_tcp_server *s) { +static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { gpr_mu_lock(&s->mu); GPR_ASSERT(!s->shutdown); @@ -315,6 +314,8 @@ error: /* event manager callback when reads are ready */ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { grpc_tcp_listener *sp = arg; + grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index, + sp->fd_index}; grpc_fd *fdobj; size_t i; @@ -363,7 +364,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { sp->server->on_accept_cb( exec_ctx, sp->server->on_accept_cb_arg, grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str), - grpc_tcp_server_ref(sp->server), sp->port_index, sp->fd_index); + &acceptor); gpr_free(name); gpr_free(addr_str); @@ -524,7 +525,8 @@ done: } } -unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, unsigned port_index) { +unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s, + unsigned port_index) { unsigned num_fds = 0; grpc_tcp_listener *sp; for (sp = s->head; sp && port_index != 0; sp = sp->next) { @@ -537,8 +539,8 @@ unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, unsigned port_index) { return num_fds; } -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index, - unsigned fd_index) { +int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index, + unsigned fd_index) { grpc_tcp_listener *sp; for (sp = s->head; sp && port_index != 0; sp = sp->next) { if (!sp->is_sibling) { @@ -585,19 +587,14 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) { return s; } -void grpc_tcp_server_set_shutdown_complete(grpc_tcp_server *s, - grpc_closure *shutdown_complete) { - s->shutdown_complete = shutdown_complete; -} - void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { if (gpr_unref(&s->refs)) { if (exec_ctx == NULL) { grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_tcp_server_destroy(&local_exec_ctx, s); + tcp_server_destroy(&local_exec_ctx, s); grpc_exec_ctx_finish(&local_exec_ctx); } else { - grpc_tcp_server_destroy(exec_ctx, s); + tcp_server_destroy(exec_ctx, s); } } } diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index 0435f5005bb..531fb003f7a 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -55,6 +55,7 @@ #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 /* one listening port */ +typedef struct grpc_tcp_listener grpc_tcp_listener; struct grpc_tcp_listener { /* This seemingly magic number comes from AcceptEx's documentation. each address buffer needs to have at least 16 more bytes at their end. */ @@ -134,8 +135,7 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) { return s; } -static void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, - grpc_tcp_server *s) { +static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { int immediately_done = 0; grpc_tcp_listener *sp; gpr_mu_lock(&s->mu); @@ -156,19 +156,14 @@ static void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, } } -void grpc_tcp_server_set_shutdown_complete(grpc_tcp_server *s, - grpc_closure *shutdown_complete) { - s->shutdown_complete = shutdown_complete; -} - void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { if (gpr_unref(&s->refs)) { if (exec_ctx == NULL) { grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_tcp_server_destroy(&local_exec_ctx, s); + tcp_server_destroy(&local_exec_ctx, s); grpc_exec_ctx_finish(&local_exec_ctx); } else { - grpc_tcp_server_destroy(exec_ctx, s); + tcp_server_destroy(exec_ctx, s); } } } @@ -300,6 +295,7 @@ failure: /* Event manager callback when reads are ready. */ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { grpc_tcp_listener *sp = arg; + grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index, 0}; SOCKET sock = sp->new_socket; grpc_winsocket_callback_info *info = &sp->socket->read_info; grpc_endpoint *ep = NULL; @@ -367,7 +363,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { /* The only time we should call our callback, is where we successfully managed to accept a connection, and created an endpoint. */ if (ep) sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, - sp->server, sp->port_index, 0); + &acceptor); /* As we were notified from the IOCP of one and exactly one accept, the former socked we created has now either been destroy or assigned to the new connection. We need to create a new one for the next @@ -495,7 +491,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, } } -unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, int port_index) { +unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s, int port_index) { grpc_tcp_listener *sp; for (sp = s->head; sp && port_index != 0; sp = sp->next, --port_index) ; @@ -506,8 +502,8 @@ unsigned grpc_tcp_server_fds_for_port(grpc_tcp_server *s, int port_index) { } } -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index, - unsigned fd_index) { +int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index, + unsigned fd_index) { grpc_tcp_listener *sp; if (fd_index != 0) { /* Windows implementation has only one fd per port_index. */ diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c index f4f3ff49218..53c8ae4c802 100644 --- a/src/core/security/server_secure_chttp2.c +++ b/src/core/security/server_secure_chttp2.c @@ -127,9 +127,7 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep, } static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp, - grpc_tcp_server *from_server, unsigned port_index, - unsigned fd_index) { - grpc_tcp_server_unref(NULL, from_server); + grpc_tcp_server_acceptor *acceptor) { grpc_server_secure_state *state = statep; state_ref(state); grpc_security_connector_do_handshake(exec_ctx, state->sc, tcp, diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c index f0b9f211b3d..cdd6d4fd969 100644 --- a/src/core/surface/server_chttp2.c +++ b/src/core/surface/server_chttp2.c @@ -53,8 +53,8 @@ static void setup_transport(grpc_exec_ctx *exec_ctx, void *server, } static void new_transport(grpc_exec_ctx *exec_ctx, void *server, - grpc_endpoint *tcp, grpc_tcp_server *tcp_server, - unsigned port_index, unsigned fd_index) { + grpc_endpoint *tcp, + grpc_tcp_server_acceptor *acceptor) { /* * Beware that the call to grpc_create_chttp2_transport() has to happen before * grpc_tcp_server_destroy(). This is fine here, but similar code @@ -66,7 +66,6 @@ static void new_transport(grpc_exec_ctx *exec_ctx, void *server, exec_ctx, grpc_server_get_channel_args(server), tcp, 0); setup_transport(exec_ctx, server, transport); grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0); - grpc_tcp_server_unref(exec_ctx, tcp_server); } /* Server callback: start listening on our ports */ @@ -82,8 +81,8 @@ static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp, static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp, grpc_closure *destroy_done) { grpc_tcp_server *tcp = tcpp; - grpc_tcp_server_set_shutdown_complete(tcp, destroy_done); grpc_tcp_server_unref(exec_ctx, tcp); + grpc_exec_ctx_enqueue(exec_ctx, destroy_done, 1); } int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c index 68832a4b4e5..bf7ef3f26be 100644 --- a/test/core/client_config/set_initial_connect_string_test.c +++ b/test/core/client_config/set_initial_connect_string_test.c @@ -79,10 +79,8 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { } static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, - grpc_tcp_server *tcp_server, unsigned port_index, - unsigned fd_index) { + grpc_tcp_server_acceptor *acceptor) { test_tcp_server *server = arg; - grpc_tcp_server_unref(NULL, tcp_server); grpc_closure_init(&on_read, handle_read, NULL); gpr_slice_buffer_init(&state.incoming_buffer); gpr_slice_buffer_init(&state.temp_incoming_buffer); diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 2791087b8a0..11848851d80 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -53,18 +53,17 @@ struct on_connect_result { }; static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, - grpc_tcp_server *tcp_server, unsigned port_index, - unsigned fd_index) { + grpc_tcp_server_acceptor *acceptor) { struct on_connect_result *result = arg; grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); - result->server_fd = grpc_tcp_server_get_fd(tcp_server, port_index, fd_index); + result->server_fd = grpc_tcp_server_port_fd( + acceptor->from_server, acceptor->port_index, acceptor->fd_index); g_nconnects++; grpc_pollset_kick(&g_pollset, NULL); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); - grpc_tcp_server_unref(exec_ctx, tcp_server); } static void test_no_op(void) { @@ -133,14 +132,14 @@ static void test_connect(int n) { addr.ss_family = AF_INET; GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len) > 0); - GPR_ASSERT(grpc_tcp_server_fds_for_port(s, 2) == 0); - GPR_ASSERT(grpc_tcp_server_fds_for_port(s, 1) == 0); - GPR_ASSERT(grpc_tcp_server_fds_for_port(s, 0) == 1); - GPR_ASSERT(grpc_tcp_server_get_fd(s, 0, 1) < 0); - GPR_ASSERT(grpc_tcp_server_get_fd(s, 0, 2) < 0); - GPR_ASSERT(grpc_tcp_server_get_fd(s, 2, 0) < 0); - GPR_ASSERT(grpc_tcp_server_get_fd(s, 1, 0) < 0); - svrfd = grpc_tcp_server_get_fd(s, 0, 0); + GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 2) == 0); + GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 1) == 0); + GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 0) == 1); + GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 1) < 0); + GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 2) < 0); + GPR_ASSERT(grpc_tcp_server_port_fd(s, 2, 0) < 0); + GPR_ASSERT(grpc_tcp_server_port_fd(s, 1, 0) < 0); + svrfd = grpc_tcp_server_port_fd(s, 0, 0); GPR_ASSERT(svrfd >= 0); GPR_ASSERT(getsockname(svrfd, (struct sockaddr *)&addr, &addr_len) == 0); GPR_ASSERT(addr_len <= sizeof(addr)); diff --git a/test/core/util/reconnect_server.c b/test/core/util/reconnect_server.c index 0db25703c4a..fa51d810d87 100644 --- a/test/core/util/reconnect_server.c +++ b/test/core/util/reconnect_server.c @@ -67,14 +67,12 @@ static void pretty_print_backoffs(reconnect_server *server) { } static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, - grpc_tcp_server *tcp_server, unsigned port_index, - unsigned fd_index) { + grpc_tcp_server_acceptor *acceptor) { char *peer; char *last_colon; reconnect_server *server = (reconnect_server *)arg; gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); timestamp_list *new_tail; - grpc_tcp_server_unref(NULL, tcp_server); peer = grpc_endpoint_get_peer(tcp); grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index cd7cd8db6af..52300c1a2d2 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -45,10 +45,17 @@ #include "src/core/iomgr/tcp_server.h" #include "test/core/util/port.h" +static void on_server_destroyed(grpc_exec_ctx *exec_ctx, void *data, + int success) { + test_tcp_server *server = data; + server->shutdown = 1; +} + void test_tcp_server_init(test_tcp_server *server, grpc_tcp_server_cb on_connect, void *user_data) { grpc_init(); server->tcp_server = NULL; + grpc_closure_init(&server->shutdown_complete, on_server_destroyed, server); server->shutdown = 0; grpc_pollset_init(&server->pollset); server->pollsets[0] = &server->pollset; @@ -65,7 +72,7 @@ void test_tcp_server_start(test_tcp_server *server, int port) { addr.sin_port = htons((uint16_t)port); memset(&addr.sin_addr, 0, sizeof(addr.sin_addr)); - server->tcp_server = grpc_tcp_server_create(NULL); + server->tcp_server = grpc_tcp_server_create(&server->shutdown_complete); port_added = grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr)); GPR_ASSERT(port_added == port); @@ -90,24 +97,14 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { grpc_exec_ctx_finish(&exec_ctx); } -static void on_server_destroyed(grpc_exec_ctx *exec_ctx, void *data, - int success) { - test_tcp_server *server = data; - server->shutdown = 1; -} - static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, int success) {} void test_tcp_server_destroy(test_tcp_server *server) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_timespec shutdown_deadline; - grpc_closure server_shutdown_cb; grpc_closure do_nothing_cb; - grpc_closure_init(&server_shutdown_cb, on_server_destroyed, server); - grpc_closure_init(&do_nothing_cb, do_nothing, NULL); - grpc_tcp_server_set_shutdown_complete(server->tcp_server, - &server_shutdown_cb); grpc_tcp_server_unref(&exec_ctx, server->tcp_server); + grpc_closure_init(&do_nothing_cb, do_nothing, NULL); shutdown_deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(5, GPR_TIMESPAN)); while (!server->shutdown && diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h index deb65eef119..5299a8f3064 100644 --- a/test/core/util/test_tcp_server.h +++ b/test/core/util/test_tcp_server.h @@ -39,6 +39,7 @@ typedef struct test_tcp_server { grpc_tcp_server *tcp_server; + grpc_closure shutdown_complete; int shutdown; grpc_pollset pollset; grpc_pollset *pollsets[1]; diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0de20a634a4..206848bf409 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -114,10 +114,13 @@ class ValgrindConfig(object): self.args = args self.allow_hashing = False - def job_spec(self, cmdline, hash_targets): + def job_spec(self, cmdline, hash_targets, timeout_seconds=None, + shortname=None, environ=None): + if shortname is None: + shortname = 'valgrind %s' % cmdline[0] return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] + self.args + cmdline, - shortname='valgrind %s' % cmdline[0], + shortname=shortname, hash_targets=None, flake_retries=5 if args.allow_flakes else 0, timeout_retries=3 if args.allow_flakes else 0) @@ -1092,4 +1095,3 @@ else: if BuildAndRunError.POST_TEST in errors: exit_code |= 4 sys.exit(exit_code) - From 03c8a9b365a4a819617909e16e9c94632de5a36d Mon Sep 17 00:00:00 2001 From: Dan Born Date: Tue, 12 Jan 2016 21:14:29 -0800 Subject: [PATCH 04/42] Put the fd_index in the same order as the sibling list. --- src/core/iomgr/tcp_server_posix.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index 3e0c5be3483..e37da136af5 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -509,11 +509,9 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, addr_len = sizeof(addr4_copy); } sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index); - if (sp2 != NULL) { - if (sp != NULL) { - sp->sibling = sp2; - } - sp2->is_sibling = 1; + if (sp2 != NULL && sp != NULL) { + sp2->sibling = sp; + sp->is_sibling = 1; } done: From 0f690721bdc977b5d506e575f5e13be6d09c57fd Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Wed, 13 Jan 2016 09:08:38 -0800 Subject: [PATCH 05/42] Move encode/decode API into census.h --- BUILD | 3 - Makefile | 1 + build.yaml | 1 - gRPC.podspec | 4 +- include/grpc/census.h | 20 ++++++- src/core/census/tag_set.c | 6 +- src/core/census/tag_set.h | 57 ------------------- src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/census/tag_set_test.c | 2 +- tools/doxygen/Doxyfile.core.internal | 1 - tools/run_tests/sources_and_headers.json | 4 -- vsprojects/vcxproj/grpc/grpc.vcxproj | 1 - vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 - .../grpc_unsecure/grpc_unsecure.vcxproj | 1 - .../grpc_unsecure.vcxproj.filters | 3 - 15 files changed, 23 insertions(+), 85 deletions(-) delete mode 100644 src/core/census/tag_set.h diff --git a/BUILD b/BUILD index ae1a69073fc..43e7ba3e181 100644 --- a/BUILD +++ b/BUILD @@ -268,7 +268,6 @@ cc_library( "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/rpc_metric_id.h", - "src/core/census/tag_set.h", "src/core/httpcli/httpcli_security_connector.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", @@ -561,7 +560,6 @@ cc_library( "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/rpc_metric_id.h", - "src/core/census/tag_set.h", "src/core/surface/init_unsecure.c", "src/core/census/grpc_context.c", "src/core/census/grpc_filter.c", @@ -1367,7 +1365,6 @@ objc_library( "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/rpc_metric_id.h", - "src/core/census/tag_set.h", ], includes = [ "include", diff --git a/Makefile b/Makefile index c359ca0375e..a1b08b414b9 100644 --- a/Makefile +++ b/Makefile @@ -8303,6 +8303,7 @@ $(BINDIR)/$(CONFIG)/tag_set_test: $(TAG_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgr endif $(OBJDIR)/$(CONFIG)/test/core/census/tag_set_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + deps_tag_set_test: $(TAG_SET_TEST_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) diff --git a/build.yaml b/build.yaml index 1177564f179..02bab578596 100644 --- a/build.yaml +++ b/build.yaml @@ -16,7 +16,6 @@ filegroups: - src/core/census/aggregation.h - src/core/census/context.h - src/core/census/rpc_metric_id.h - - src/core/census/tag_set.h src: - src/core/census/context.c - src/core/census/initialize.c diff --git a/gRPC.podspec b/gRPC.podspec index 4acf2d7e1ba..cde97ac661f 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -272,7 +272,6 @@ Pod::Spec.new do |s| 'src/core/census/aggregation.h', 'src/core/census/context.h', 'src/core/census/rpc_metric_id.h', - 'src/core/census/tag_set.h', 'include/grpc/grpc_security.h', 'include/grpc/byte_buffer.h', 'include/grpc/byte_buffer_reader.h', @@ -571,8 +570,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', 'src/core/census/context.h', - 'src/core/census/rpc_metric_id.h', - 'src/core/census/tag_set.h' + 'src/core/census/rpc_metric_id.h' ss.header_mappings_dir = '.' # This isn't officially supported in Cocoapods. We've asked for an alternative: diff --git a/include/grpc/census.h b/include/grpc/census.h index 908bcc96e2e..2a62f4daca5 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -339,7 +339,7 @@ typedef struct { const char *key; const char *value; size_t value_len; - gpr_uint8 flags; + uint8_t flags; } census_tag; /* Tag flags. */ @@ -386,6 +386,22 @@ int census_tag_set_get_tag_by_index(const census_tag_set *tags, int index, int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, census_tag *tag); +/* Encode to-be-propagated non-binary tags from a tag set into a memory + buffer. The total number of bytes used in the buffer is returned. If the + buffer is too small to contain the encoded tag set, then 0 is returned. */ +size_t census_tag_set_encode_propagated(const census_tag_set *tags, + char *buffer, size_t buf_size); + +/* Encode to-be-propagated binary tags from a tag set into a memory + buffer. The total number of bytes used in the buffer is returned. If the + buffer is too small to contain the encoded tag set, then 0 is returned. */ +size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, + char *buffer, size_t buf_size); + +/* Decode tag set buffers encoded with census_tag_set_encode_*(). */ +census_tag_set *census_tag_set_decode(const char *buffer, size_t size, + const char *bin_buffer, size_t bin_size); + /* Get a contexts tag set. */ census_tag_set *census_context_tag_set(census_context *context); diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 80c209031f4..fcfe67afad3 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,17 +31,13 @@ * */ -#include "tag_set.h" - #include #include #include #include -#include #include #include #include -#include "src/core/support/murmur_hash.h" #include "src/core/support/string.h" // Functions in this file support the public tag_set API, as well as diff --git a/src/core/census/tag_set.h b/src/core/census/tag_set.h deleted file mode 100644 index 9eec0ad4384..00000000000 --- a/src/core/census/tag_set.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPC_INTERNAL_CORE_CENSUS_TAG_SET_H -#define GRPC_INTERNAL_CORE_CENSUS_TAG_SET_H - -#include -#include -#include - -/* Encode to-be-propagated tags from a tag set into a memory buffer. The total - number of bytes used in the buffer is returned. If the buffer is too small - to contain the encoded tag set, then 0 is returned. */ -size_t census_tag_set_encode_propagated(const census_tag_set *tags, - char *buffer, size_t buf_size); - -/* Encode to-be-propagated binary tags from a tag set into a memory - buffer. The total number of bytes used in the buffer is returned. If the - buffer is too small to contain the encoded tag set, then 0 is returned. */ -size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, - char *buffer, size_t buf_size); - -/* Decode tag set buffers encoded with census_tag_set_encode_*(). */ -census_tag_set *census_tag_set_decode(const char *buffer, size_t size, - const char *bin_buffer, size_t bin_size); - -#endif /* GRPC_INTERNAL_CORE_CENSUS_TAG_SET_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 63cf0a4c74c..f5b8b897f9d 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -220,6 +220,7 @@ CORE_SOURCE_FILES = [ 'src/core/census/context.c', 'src/core/census/initialize.c', 'src/core/census/operation.c', + 'src/core/census/tag_set.c', 'src/core/census/tracing.c', 'src/boringssl/err_data.c', 'third_party/boringssl/crypto/aes/aes.c', diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index d6a7e453346..44ecb2819ce 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -33,7 +33,7 @@ // Test census_tag_set functions, including encoding/decoding -#include "src/core/census/tag_set.h" +#include #include #include #include diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ac57dd03b7e..222ad3c6a22 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -897,7 +897,6 @@ src/core/transport/transport_impl.h \ src/core/census/aggregation.h \ src/core/census/context.h \ src/core/census/rpc_metric_id.h \ -src/core/census/tag_set.h \ src/core/httpcli/httpcli_security_connector.c \ src/core/security/base64.c \ src/core/security/client_auth_filter.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 5e474a8b7b1..4067863f89e 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2874,7 +2874,6 @@ "src/core/census/context.h", "src/core/census/grpc_filter.h", "src/core/census/rpc_metric_id.h", - "src/core/census/tag_set.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", "src/core/channel/client_channel.h", @@ -3022,7 +3021,6 @@ "src/core/census/operation.c", "src/core/census/rpc_metric_id.h", "src/core/census/tag_set.c", - "src/core/census/tag_set.h", "src/core/census/tracing.c", "src/core/channel/channel_args.c", "src/core/channel/channel_args.h", @@ -3389,7 +3387,6 @@ "src/core/census/context.h", "src/core/census/grpc_filter.h", "src/core/census/rpc_metric_id.h", - "src/core/census/tag_set.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", "src/core/channel/client_channel.h", @@ -3522,7 +3519,6 @@ "src/core/census/operation.c", "src/core/census/rpc_metric_id.h", "src/core/census/tag_set.c", - "src/core/census/tag_set.h", "src/core/census/tracing.c", "src/core/channel/channel_args.c", "src/core/channel/channel_args.h", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 037b8ff2224..b143de9d86d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -404,7 +404,6 @@ - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 0ddbd5e46d5..93848293c63 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -869,9 +869,6 @@ src\core\census - - src\core\census - diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index a3532e5d05e..dae8edf17f7 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -382,7 +382,6 @@ - diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index f63e3012e75..1f850b26292 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -764,9 +764,6 @@ src\core\census - - src\core\census - From a237796233b601e7a5bf1af79c68b1cc269a6f87 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Wed, 13 Jan 2016 11:09:16 -0800 Subject: [PATCH 06/42] Always compress tags --- src/core/census/tag_set.c | 218 ++++++++++++++------------------ test/core/census/tag_set_test.c | 13 -- 2 files changed, 93 insertions(+), 138 deletions(-) diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index fcfe67afad3..e879ac06068 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -59,13 +59,12 @@ // to handle endian-ness conversions. // * Keep all tag information (keys/values/flags) in a single memory buffer, // that can be directly copied to the wire. This makes iteration by index -// somewhat less efficient. +// somewhat less efficient. If it becomes a problem, we could consider +// building an index at tag_set creation. // * Binary tags are encoded seperately from non-binary tags. There are a // primarily because non-binary tags are far more likely to be repeated // across multiple RPC calls, so are more efficiently cached and // compressed in any metadata schemes. -// * deleted/modified tags are kept in memory, just marked with a deleted -// flag. This enables faster processing TODO: benchmark this // * all lengths etc. are restricted to one byte. This eliminates endian // issues. @@ -74,7 +73,8 @@ struct tag_set { int ntags; // number of tags. int ntags_alloc; // ntags + number of deleted tags (total number of tags - // in all of kvm). + // in all of kvm). This will always be == ntags, except + // during the process of building a new tag set. size_t kvm_size; // number of bytes allocated for key/value storage. size_t kvm_used; // number of bytes of used key/value memory char *kvm; // key/value memory. Consists of repeated entries of: @@ -138,28 +138,17 @@ static void tag_set_copy(struct tag_set *to, const struct tag_set *from) { memcpy(to->kvm, from->kvm, to->kvm_used); } -// We may want to keep information about a deleted tag for a short time, -// in case we can reuse the space (same tag is reinserted). This structure -// is used for that purpose. -struct deleted_tag_info { - struct raw_tag raw; // raw tag information. - uint8_t *raw_flags_p; // pointer to original flags - struct tag_set *tags; // the tag set from which tag was deleted. -}; - -// Delete a tag from a tag set, if it exists. Returns true if the tag was -// originally present (and is now deleted), false if it wasn't. -static bool tag_set_delete_tag(struct tag_set *tags, - struct deleted_tag_info *dtag, const char *key, +// Delete a tag from a tag set, if it exists (returns true it it did). +static bool tag_set_delete_tag(struct tag_set *tags, const char *key, size_t key_len) { char *kvp = tags->kvm; for (int i = 0; i < tags->ntags_alloc; i++) { - dtag->raw_flags_p = (uint8_t *)(kvp + FLAG_OFFSET); - kvp = decode_tag(&dtag->raw, kvp, 0); - if (CENSUS_TAG_IS_DELETED(dtag->raw.flags)) continue; - if ((key_len == dtag->raw.key_len) && - (memcmp(key, dtag->raw.key, key_len) == 0)) { - *(dtag->raw_flags_p) |= CENSUS_TAG_DELETED; + uint8_t *flags = (uint8_t *)(kvp + FLAG_OFFSET); + struct raw_tag tag; + kvp = decode_tag(&tag, kvp, 0); + if (CENSUS_TAG_IS_DELETED(tag.flags)) continue; + if ((key_len == tag.key_len) && (memcmp(key, tag.key, key_len) == 0)) { + *flags |= CENSUS_TAG_DELETED; tags->ntags--; return true; } @@ -167,46 +156,34 @@ static bool tag_set_delete_tag(struct tag_set *tags, return false; } -// Delete a tag from a tag set. If the tag is found in any of the underlying -// tag sets, *and* that tag set corresponds to the one in which the tag (if -// later inserted) would be placed, then fills in dtag, and returns true. -// Returns false otherwise. This information is later used to optimize the -// placement of the tag if the value space can be reused, effectively -// "undeleting" the tag. -static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, - size_t key_len, struct deleted_tag_info *dtag) { +// Delete a tag from a tag set. +static void cts_delete_tag(census_tag_set *tags, const census_tag *tag, + size_t key_len) { // use the to-be-deleted tag flags as a hint to determine the order in which // we delete from the underlying tag sets. if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { if (CENSUS_TAG_IS_BINARY(tag->flags)) { - if (tag_set_delete_tag(&tags->propagated_binary_tags, dtag, tag->key, + if (tag_set_delete_tag(&tags->propagated_binary_tags, tag->key, key_len)) { - dtag->tags = &tags->propagated_binary_tags; - return true; + return; } - if (tag_set_delete_tag(&tags->propagated_tags, dtag, tag->key, key_len)) - return false; - tag_set_delete_tag(&tags->local_tags, dtag, tag->key, key_len); + if (tag_set_delete_tag(&tags->propagated_tags, tag->key, key_len)) return; + tag_set_delete_tag(&tags->local_tags, tag->key, key_len); } else { - if (tag_set_delete_tag(&tags->propagated_tags, dtag, tag->key, key_len)) { - dtag->tags = &tags->propagated_tags; - return true; + if (tag_set_delete_tag(&tags->propagated_tags, tag->key, key_len)) { + return; } - if (tag_set_delete_tag(&tags->propagated_binary_tags, dtag, tag->key, - key_len)) - return false; - tag_set_delete_tag(&tags->local_tags, dtag, tag->key, key_len); + if (tag_set_delete_tag(&tags->propagated_binary_tags, tag->key, key_len)) + return; + tag_set_delete_tag(&tags->local_tags, tag->key, key_len); } } else { - if (tag_set_delete_tag(&tags->local_tags, dtag, tag->key, key_len)) { - dtag->tags = &tags->local_tags; - return true; + if (tag_set_delete_tag(&tags->local_tags, tag->key, key_len)) { + return; } - if (tag_set_delete_tag(&tags->propagated_tags, dtag, tag->key, key_len)) - return false; - tag_set_delete_tag(&tags->propagated_binary_tags, dtag, tag->key, key_len); + if (tag_set_delete_tag(&tags->propagated_tags, tag->key, key_len)) return; + tag_set_delete_tag(&tags->propagated_binary_tags, tag->key, key_len); } - return false; } // Add a tag to a tag set. @@ -218,6 +195,7 @@ static void tag_set_add_tag(struct tag_set *tags, const census_tag *tag, return; } if (tags->kvm_used + tag_size > tags->kvm_size) { + // allocate new memory if needed tags->kvm_size += 2 * CENSUS_MAX_TAG_KV_LEN + TAG_HEADER_SIZE; char *new_kvm = gpr_malloc(tags->kvm_size); memcpy(new_kvm, tags->kvm, tags->kvm_used); @@ -242,28 +220,60 @@ static void tag_set_add_tag(struct tag_set *tags, const census_tag *tag, static void cts_add_tag(census_tag_set *tags, const census_tag *tag, size_t key_len) { // first delete the tag if it is already present - struct deleted_tag_info dtag; - bool deleted_match = cts_delete_tag(tags, tag, key_len, &dtag); + cts_delete_tag(tags, tag, key_len); if (tag->value != NULL && tag->value_len != 0) { - if (deleted_match && tag->value_len == dtag.raw.value_len) { - // if we have a close match for tag being added to one just deleted, - // only need to modify value and flags. - memcpy(dtag.raw.value, tag->value, tag->value_len); - *dtag.raw_flags_p = (tag->flags & (CENSUS_TAG_PROPAGATE | - CENSUS_TAG_STATS | CENSUS_TAG_BINARY)); - dtag.tags->ntags++; - } else { - if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { - if (CENSUS_TAG_IS_BINARY(tag->flags)) { - tag_set_add_tag(&tags->propagated_binary_tags, tag, key_len); - } else { - tag_set_add_tag(&tags->propagated_tags, tag, key_len); - } + if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { + if (CENSUS_TAG_IS_BINARY(tag->flags)) { + tag_set_add_tag(&tags->propagated_binary_tags, tag, key_len); } else { - tag_set_add_tag(&tags->local_tags, tag, key_len); + tag_set_add_tag(&tags->propagated_tags, tag, key_len); + } + } else { + tag_set_add_tag(&tags->local_tags, tag, key_len); + } + } +} + +// Remove any deleted tags from the tag set. Basic algorithm: +// 1) Walk through tag set to find first deleted tag. Record where it is. +// 2) Find the next not-deleted tag. Copy all of kvm from there to the end +// "over" the deleted tags +// 3) repeat #1 and #2 until we have seen all tags +// 4) if we are still looking for a not-deleted tag, then all the end portion +// of the kvm is deleted. Just reduce the used amount of memory by the +// appropriate amount. +static void tag_set_compress(struct tag_set *tags) { + if (tags->ntags == tags->ntags_alloc) return; + bool find_deleted = true; // are we looking for deleted tags? + char *kvp = tags->kvm; + char *dbase; // record location of deleted tag + for (int i = 0; i < tags->ntags_alloc; i++) { + struct raw_tag tag; + char *next_kvp = decode_tag(&tag, kvp, 0); + if (find_deleted) { + if (CENSUS_TAG_IS_DELETED(tag.flags)) { + dbase = kvp; + find_deleted = false; + } + } else { + if (!CENSUS_TAG_IS_DELETED(tag.flags)) { + ptrdiff_t reduce = kvp - dbase; // #bytes in deleted tags + GPR_ASSERT(reduce > 0); + ptrdiff_t copy_size = tags->kvm + tags->kvm_used - kvp; + GPR_ASSERT(copy_size > 0); + memmove(dbase, kvp, (size_t)copy_size); + tags->kvm_used -= (size_t)reduce; + next_kvp -= reduce; + find_deleted = true; } } + kvp = next_kvp; } + if (!find_deleted) { + GPR_ASSERT(dbase > tags->kvm); + tags->kvm_used = (size_t)(dbase - tags->kvm); + } + tags->ntags_alloc = tags->ntags; } census_tag_set *census_tag_set_create(const census_tag_set *base, @@ -286,6 +296,9 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, cts_add_tag(new_ts, tag, key_len); } } + tag_set_compress(&new_ts->propagated_tags); + tag_set_compress(&new_ts->propagated_binary_tags); + tag_set_compress(&new_ts->local_tags); return new_ts; } @@ -307,21 +320,15 @@ static void tag_set_get_tag_by_index(const struct tag_set *tags, int index, census_tag *tag) { GPR_ASSERT(index < tags->ntags); char *kvp = tags->kvm; - for (;;) { - struct raw_tag raw; + struct raw_tag raw; + kvp = decode_tag(&raw, kvp, 0); + for (int i = 0; i < index; i++) { kvp = decode_tag(&raw, kvp, 0); - if (CENSUS_TAG_IS_DELETED(raw.flags)) { - continue; - } else if (index == 0) { - tag->key = raw.key; - tag->value = raw.value; - tag->value_len = raw.value_len; - tag->flags = raw.flags; - return; - } - index--; } - // NOT REACHED + tag->key = raw.key; + tag->value = raw.value; + tag->value_len = raw.value_len; + tag->flags = raw.flags; } int census_tag_set_get_tag_by_index(const census_tag_set *tags, int index, @@ -350,9 +357,7 @@ static bool tag_set_get_tag_by_key(const struct tag_set *tags, const char *key, char *kvp = tags->kvm; for (int i = 0; i < tags->ntags; i++) { struct raw_tag raw; - do { - kvp = decode_tag(&raw, kvp, 0); - } while (CENSUS_TAG_IS_DELETED(raw.flags)); + kvp = decode_tag(&raw, kvp, 0); if (key_len == raw.key_len && memcmp(raw.key, key, key_len) == 0) { tag->key = raw.key; tag->value = raw.value; @@ -391,42 +396,16 @@ int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, // expansion. // 2 1 number of bytes in each tag header. // 3 1 ntags value from tag set. -// 4 1 ntags_alloc value from tag set. // // This is followed by the key/value memory from struct tag_set. #define ENCODED_VERSION 0 // Version number -#define ENCODED_HEADER_SIZE 5 // size of tag set header - -// Pack a tag set into as few bytes as possible (eliding deleted tags). Assumes -// header is already generated. -static size_t tag_set_encode_packed(const struct tag_set *tags, char *buffer, - size_t buf_size) { - size_t encoded_size = 0; - char *kvp = tags->kvm; - for (int i = 0; i < tags->ntags_alloc; i++) { - struct raw_tag raw; - char *base = kvp; - kvp = decode_tag(&raw, kvp, 0); - size_t tag_size = - TAG_HEADER_SIZE + (size_t)raw.key_len + (size_t)raw.value_len; - if (!(CENSUS_TAG_IS_DELETED(raw.flags))) { - if (tag_size > buf_size) { - return 0; - } - memcpy(buffer, base, tag_size); - buffer += tag_size; - encoded_size += tag_size; - buf_size -= tag_size; - } - } - return encoded_size; -} +#define ENCODED_HEADER_SIZE 4 // size of tag set header // Encode a tag set. Returns 0 if buffer is too small. static size_t tag_set_encode(const struct tag_set *tags, char *buffer, size_t buf_size) { - if (buf_size < ENCODED_HEADER_SIZE) { + if (buf_size < ENCODED_HEADER_SIZE + tags->kvm_used) { return 0; } buf_size -= ENCODED_HEADER_SIZE; @@ -435,18 +414,8 @@ static size_t tag_set_encode(const struct tag_set *tags, char *buffer, *buffer++ = (char)TAG_HEADER_SIZE; *buffer++ = (char)tags->ntags; if (tags->ntags == 0) { - *buffer = (char)tags->ntags; return ENCODED_HEADER_SIZE; } - if (buf_size < tags->kvm_used || tags->ntags_alloc > CENSUS_MAX_TAGS) { - *buffer++ = (char)tags->ntags; - size_t enc_size = tag_set_encode_packed(tags, buffer, buf_size); - if (enc_size == 0) { - return 0; - } - return ENCODED_HEADER_SIZE + enc_size; - } - *buffer++ = (char)tags->ntags_alloc; memcpy(buffer, tags->kvm, tags->kvm_used); return ENCODED_HEADER_SIZE + tags->kvm_used; } @@ -467,7 +436,7 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, uint8_t version = (uint8_t)(*buffer++); uint8_t header_size = (uint8_t)(*buffer++); uint8_t tag_header_size = (uint8_t)(*buffer++); - tags->ntags = (int)(*buffer++); + tags->ntags = tags->ntags_alloc = (int)(*buffer++); if (tags->ntags == 0) { tags->ntags_alloc = 0; tags->kvm_size = 0; @@ -475,7 +444,6 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, tags->kvm = NULL; return; } - tags->ntags_alloc = (uint8_t)(*buffer++); if (header_size != ENCODED_HEADER_SIZE) { GPR_ASSERT(version != ENCODED_VERSION); GPR_ASSERT(ENCODED_HEADER_SIZE < header_size); @@ -490,7 +458,7 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, GPR_ASSERT(version != ENCODED_VERSION); GPR_ASSERT(tag_header_size > TAG_HEADER_SIZE); char *kvp = tags->kvm; - for (int i = 0; i < tags->ntags_alloc; i++) { + for (int i = 0; i < tags->ntags; i++) { memcpy(kvp, buffer, TAG_HEADER_SIZE); kvp += header_size; struct raw_tag raw; diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index 44ecb2819ce..4c286b3e4f7 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -367,19 +367,6 @@ static void complex_encode_decode_test(void) { GPR_ASSERT(census_tag_set_ntags(cts3) == 2); GPR_ASSERT(validate_tag(cts3, &basic_tags[4])); GPR_ASSERT(validate_tag(cts3, &modify_tags[8])); - // Now force tag set to be in smaller space - census_tag_set_destroy(cts3); - size_t nb1 = census_tag_set_encode_propagated(cts2, buf1, b1 - 1); - GPR_ASSERT(nb1 != 0); - GPR_ASSERT(nb1 < b1); - size_t nb2 = census_tag_set_encode_propagated_binary(cts2, buf2, b2 - 1); - GPR_ASSERT(nb2 != 0); - GPR_ASSERT(nb2 < b2); - cts3 = census_tag_set_decode(buf1, nb1, buf2, nb2); - GPR_ASSERT(cts3 != NULL); - GPR_ASSERT(census_tag_set_ntags(cts3) == 2); - GPR_ASSERT(validate_tag(cts3, &basic_tags[4])); - GPR_ASSERT(validate_tag(cts3, &modify_tags[8])); census_tag_set_destroy(cts3); census_tag_set_destroy(cts2); census_tag_set_destroy(cts); From 725ee28af8924670a0a9974094b5ea773c519419 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 13 Jan 2016 13:14:56 -0800 Subject: [PATCH 07/42] Extend Copyrights to 2016 --- src/core/iomgr/tcp_server.h | 2 +- src/core/iomgr/tcp_server_posix.c | 2 +- src/core/iomgr/tcp_server_windows.c | 2 +- src/core/security/server_secure_chttp2.c | 2 +- src/core/surface/server_chttp2.c | 2 +- test/core/client_config/set_initial_connect_string_test.c | 2 +- test/core/iomgr/tcp_server_posix_test.c | 2 +- test/core/util/reconnect_server.c | 2 +- test/core/util/test_tcp_server.c | 2 +- test/core/util/test_tcp_server.h | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h index e4fd0999a9f..5199c25eed5 100644 --- a/src/core/iomgr/tcp_server.h +++ b/src/core/iomgr/tcp_server.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index e37da136af5..6a22c08edba 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index 531fb003f7a..9a831722df0 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c index 53c8ae4c802..08713fceaf8 100644 --- a/src/core/security/server_secure_chttp2.c +++ b/src/core/security/server_secure_chttp2.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c index cdd6d4fd969..6e21d2dcd77 100644 --- a/src/core/surface/server_chttp2.c +++ b/src/core/surface/server_chttp2.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c index bf7ef3f26be..33cab715b2a 100644 --- a/test/core/client_config/set_initial_connect_string_test.c +++ b/test/core/client_config/set_initial_connect_string_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 11848851d80..f1c7cfd02fe 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/util/reconnect_server.c b/test/core/util/reconnect_server.c index fa51d810d87..57225aa8a33 100644 --- a/test/core/util/reconnect_server.c +++ b/test/core/util/reconnect_server.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 52300c1a2d2..aaba7be3569 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h index 5299a8f3064..51119cf6c80 100644 --- a/test/core/util/test_tcp_server.h +++ b/test/core/util/test_tcp_server.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 50ef3042a16a0db47449840b25a0a0cc837d8d78 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 13 Jan 2016 13:35:33 -0800 Subject: [PATCH 08/42] Fix windows compilation errors --- src/core/iomgr/tcp_server_windows.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index 9a831722df0..aad2e5f358a 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -125,7 +125,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { s->head = sp->next; sp->next = NULL; grpc_winsocket_destroy(sp->socket); - grpc_tcp_listener_unref(sp); + gpr_free(sp); } gpr_free(s); } @@ -417,7 +417,6 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock, sp->new_socket = INVALID_SOCKET; sp->port = port; sp->port_index = port_index; - gpr_ref_init(&sp->refs, 1); grpc_closure_init(&sp->on_accept, on_accept, sp); GPR_ASSERT(sp->socket); gpr_mu_unlock(&s->mu); @@ -491,7 +490,8 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, } } -unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s, int port_index) { +unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s, + unsigned port_index) { grpc_tcp_listener *sp; for (sp = s->head; sp && port_index != 0; sp = sp->next, --port_index) ; From 601acadc91f0b469af0cc763824381a202d73059 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 13 Jan 2016 19:52:58 -0800 Subject: [PATCH 09/42] Test a server with multiple ports --- test/core/iomgr/tcp_server_posix_test.c | 177 +++++++++++++++++------- 1 file changed, 127 insertions(+), 50 deletions(-) diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index f1c7cfd02fe..b336002429a 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -33,11 +33,13 @@ #include "src/core/iomgr/tcp_server.h" #include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/sockaddr_utils.h" #include #include #include #include "test/core/util/test_config.h" +#include #include #include #include @@ -48,18 +50,23 @@ static grpc_pollset g_pollset; static int g_nconnects = 0; -struct on_connect_result { +typedef struct on_connect_result { + unsigned port_index; + unsigned fd_index; int server_fd; -}; +} on_connect_result; + +static on_connect_result g_result = {0, 0, -1}; static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_tcp_server_acceptor *acceptor) { - struct on_connect_result *result = arg; grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); - result->server_fd = grpc_tcp_server_port_fd( + g_result.port_index = acceptor->port_index; + g_result.fd_index = acceptor->fd_index; + g_result.server_fd = grpc_tcp_server_port_fd( acceptor->from_server, acceptor->port_index, acceptor->fd_index); g_nconnects++; grpc_pollset_kick(&g_pollset, NULL); @@ -114,68 +121,138 @@ static void test_no_op_with_port_and_start(void) { grpc_exec_ctx_finish(&exec_ctx); } -static void test_connect(int n) { +static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, + socklen_t remote_len, on_connect_result *result) { + gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); + int clifd = socket(remote->sa_family, SOCK_STREAM, 0); + int nconnects_before; + + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + nconnects_before = g_nconnects; + g_result.server_fd = -1; + GPR_ASSERT(clifd >= 0); + gpr_log(GPR_DEBUG, "start connect"); + GPR_ASSERT(connect(clifd, remote, remote_len) == 0); + gpr_log(GPR_DEBUG, "wait"); + while (g_nconnects == nconnects_before && + gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { + grpc_pollset_worker worker; + grpc_pollset_work(exec_ctx, &g_pollset, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), deadline); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_exec_ctx_finish(exec_ctx); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + } + gpr_log(GPR_DEBUG, "wait done"); + GPR_ASSERT(g_nconnects == nconnects_before + 1); + close(clifd); + *result = g_result; + + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); +} + +/* grpc_tcp_server_add_port() will not pick a second port randomly. We must find + one by binding one randomly, closing the socket, and returning the port + number. This might fail because of races. */ +static int unused_port() { + int s = socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + GPR_ASSERT(s >= 0); + memset(&addr, 0, sizeof(addr)); + addr.ss_family = AF_INET; + if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) != 0) { + gpr_log(GPR_ERROR, "bind: %s", strerror(errno)); + abort(); + } + GPR_ASSERT(getsockname(s, (struct sockaddr *)&addr, &addr_len) == 0); + GPR_ASSERT(addr_len <= sizeof(addr)); + close(s); + return ntohs(((struct sockaddr_in *)&addr)->sin_port); +} + +/* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for + the same port should be tested. */ +static void test_connect(unsigned n) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; struct sockaddr_storage addr; + struct sockaddr_storage addr1; socklen_t addr_len = sizeof(addr); - int svrfd, clifd; + unsigned svr_fd_count; + int svr_port; + unsigned svr1_fd_count; + int svr1_port; grpc_tcp_server *s = grpc_tcp_server_create(NULL); - int nconnects_before; - gpr_timespec deadline; grpc_pollset *pollsets[1]; - int i; - struct on_connect_result result; + unsigned i; LOG_TEST("test_connect"); gpr_log(GPR_INFO, "clients=%d", n); - memset(&addr, 0, sizeof(addr)); - addr.ss_family = AF_INET; - GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len) > - 0); + memset(&addr1, 0, sizeof(addr1)); + addr.ss_family = addr1.ss_family = AF_INET; + svr_port = grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len); + GPR_ASSERT(svr_port > 0); + svr1_port = unused_port(); + grpc_sockaddr_set_port((struct sockaddr *)&addr1, svr1_port); + GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr1, addr_len) == + svr1_port); + + /* Bad port_index. */ GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 2) == 0); - GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 1) == 0); - GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 0) == 1); - GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 1) < 0); - GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 2) < 0); GPR_ASSERT(grpc_tcp_server_port_fd(s, 2, 0) < 0); - GPR_ASSERT(grpc_tcp_server_port_fd(s, 1, 0) < 0); - svrfd = grpc_tcp_server_port_fd(s, 0, 0); - GPR_ASSERT(svrfd >= 0); - GPR_ASSERT(getsockname(svrfd, (struct sockaddr *)&addr, &addr_len) == 0); - GPR_ASSERT(addr_len <= sizeof(addr)); - pollsets[0] = &g_pollset; - grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, &result); + /* Bad fd_index. */ + GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 100) < 0); + GPR_ASSERT(grpc_tcp_server_port_fd(s, 1, 100) < 0); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + /* Got at least one fd per port. */ + svr_fd_count = grpc_tcp_server_port_fd_count(s, 0); + GPR_ASSERT(svr_fd_count >= 1); + svr1_fd_count = grpc_tcp_server_port_fd_count(s, 1); + GPR_ASSERT(svr1_fd_count >= 1); - for (i = 0; i < n; i++) { - deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); - nconnects_before = g_nconnects; - result.server_fd = -1; - clifd = socket(addr.ss_family, SOCK_STREAM, 0); - GPR_ASSERT(clifd >= 0); - gpr_log(GPR_DEBUG, "start connect"); - GPR_ASSERT(connect(clifd, (struct sockaddr *)&addr, addr_len) == 0); - - gpr_log(GPR_DEBUG, "wait"); - while (g_nconnects == nconnects_before && - gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { - grpc_pollset_worker worker; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); - grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + for (i = 0; i < svr_fd_count; ++i) { + int fd = grpc_tcp_server_port_fd(s, 0, i); + GPR_ASSERT(fd >= 0); + if (i == 0) { + GPR_ASSERT(getsockname(fd, (struct sockaddr *)&addr, &addr_len) == 0); + GPR_ASSERT(addr_len <= sizeof(addr)); + } + } + for (i = 0; i < svr1_fd_count; ++i) { + int fd = grpc_tcp_server_port_fd(s, 1, i); + GPR_ASSERT(fd >= 0); + if (i == 0) { + GPR_ASSERT(getsockname(fd, (struct sockaddr *)&addr1, &addr_len) == 0); + GPR_ASSERT(addr_len <= sizeof(addr1)); } - gpr_log(GPR_DEBUG, "wait done"); - - GPR_ASSERT(g_nconnects == nconnects_before + 1); - close(clifd); - GPR_ASSERT(svrfd == result.server_fd); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + pollsets[0] = &g_pollset; + grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL); + + for (i = 0; i < n; i++) { + on_connect_result result = {0, 0, -1}; + int svr_fd; + tcp_connect(&exec_ctx, (struct sockaddr *)&addr, addr_len, &result); + GPR_ASSERT(result.server_fd >= 0); + svr_fd = result.server_fd; + GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) + == result.server_fd); + GPR_ASSERT(result.port_index == 0); + GPR_ASSERT(result.fd_index < svr_fd_count); + + result.port_index = 0; + result.fd_index = 0; + result.server_fd = -1; + tcp_connect(&exec_ctx, (struct sockaddr *)&addr1, addr_len, &result); + GPR_ASSERT(result.server_fd >= 0); + GPR_ASSERT(result.server_fd != svr_fd); + GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) + == result.server_fd); + GPR_ASSERT(result.port_index == 1); + GPR_ASSERT(result.fd_index < svr_fd_count); + } grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); From a78ca38e108610672b38cf0ecb0507d167ee25ce Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 14 Jan 2016 13:02:09 -0800 Subject: [PATCH 10/42] clang reformat. --- src/core/iomgr/tcp_server_windows.c | 5 +++-- test/core/iomgr/tcp_server_posix_test.c | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index aad2e5f358a..46617a6c652 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -362,8 +362,9 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { /* The only time we should call our callback, is where we successfully managed to accept a connection, and created an endpoint. */ - if (ep) sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, - &acceptor); + if (ep) + sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, + &acceptor); /* As we were notified from the IOCP of one and exactly one accept, the former socked we created has now either been destroy or assigned to the new connection. We need to create a new one for the next diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index b336002429a..44874a7ad51 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -237,8 +237,8 @@ static void test_connect(unsigned n) { tcp_connect(&exec_ctx, (struct sockaddr *)&addr, addr_len, &result); GPR_ASSERT(result.server_fd >= 0); svr_fd = result.server_fd; - GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) - == result.server_fd); + GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) == + result.server_fd); GPR_ASSERT(result.port_index == 0); GPR_ASSERT(result.fd_index < svr_fd_count); @@ -248,8 +248,8 @@ static void test_connect(unsigned n) { tcp_connect(&exec_ctx, (struct sockaddr *)&addr1, addr_len, &result); GPR_ASSERT(result.server_fd >= 0); GPR_ASSERT(result.server_fd != svr_fd); - GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) - == result.server_fd); + GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) == + result.server_fd); GPR_ASSERT(result.port_index == 1); GPR_ASSERT(result.fd_index < svr_fd_count); } From d890a217da23eb141f38ba1c15e404d634a7bcb1 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 14 Jan 2016 15:00:21 -0800 Subject: [PATCH 11/42] Fix port allocation in test. --- test/core/iomgr/tcp_server_posix_test.c | 30 +++++-------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 44874a7ad51..564693367d3 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -37,6 +37,7 @@ #include #include #include +#include "test/core/util/port.h" #include "test/core/util/test_config.h" #include @@ -151,29 +152,9 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } -/* grpc_tcp_server_add_port() will not pick a second port randomly. We must find - one by binding one randomly, closing the socket, and returning the port - number. This might fail because of races. */ -static int unused_port() { - int s = socket(AF_INET, SOCK_STREAM, 0); - struct sockaddr_storage addr; - socklen_t addr_len = sizeof(addr); - GPR_ASSERT(s >= 0); - memset(&addr, 0, sizeof(addr)); - addr.ss_family = AF_INET; - if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) != 0) { - gpr_log(GPR_ERROR, "bind: %s", strerror(errno)); - abort(); - } - GPR_ASSERT(getsockname(s, (struct sockaddr *)&addr, &addr_len) == 0); - GPR_ASSERT(addr_len <= sizeof(addr)); - close(s); - return ntohs(((struct sockaddr_in *)&addr)->sin_port); -} - /* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for the same port should be tested. */ -static void test_connect(unsigned n) { +static void test_connect(int svr1_port, unsigned n) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; struct sockaddr_storage addr; struct sockaddr_storage addr1; @@ -181,7 +162,6 @@ static void test_connect(unsigned n) { unsigned svr_fd_count; int svr_port; unsigned svr1_fd_count; - int svr1_port; grpc_tcp_server *s = grpc_tcp_server_create(NULL); grpc_pollset *pollsets[1]; unsigned i; @@ -192,7 +172,6 @@ static void test_connect(unsigned n) { addr.ss_family = addr1.ss_family = AF_INET; svr_port = grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len); GPR_ASSERT(svr_port > 0); - svr1_port = unused_port(); grpc_sockaddr_set_port((struct sockaddr *)&addr1, svr1_port); GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr1, addr_len) == svr1_port); @@ -265,6 +244,7 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) { int main(int argc, char **argv) { grpc_closure destroyed; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + int svr1_port = grpc_pick_unused_port_or_die(); grpc_test_init(argc, argv); grpc_iomgr_init(); grpc_pollset_init(&g_pollset); @@ -273,8 +253,8 @@ int main(int argc, char **argv) { test_no_op_with_start(); test_no_op_with_port(); test_no_op_with_port_and_start(); - test_connect(1); - test_connect(10); + test_connect(svr1_port, 1); + test_connect(svr1_port, 10); grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); From b13a69da41ddad7880f409c53d1f55982ee79ac5 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 14 Jan 2016 16:53:18 -0800 Subject: [PATCH 12/42] tcp_server_posix_test fixes. Use grpc_shutdown() instead of grpc_iomgr_shutdown() to prevent grpc_pick_unused_port_or_die() from inappropriately destroying global state. Fix port allocation issues. --- test/core/iomgr/tcp_server_posix_test.c | 54 +++++++++++++++++-------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 564693367d3..0f92166ab26 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -34,6 +34,7 @@ #include "src/core/iomgr/tcp_server.h" #include "src/core/iomgr/iomgr.h" #include "src/core/iomgr/sockaddr_utils.h" +#include #include #include #include @@ -52,12 +53,30 @@ static grpc_pollset g_pollset; static int g_nconnects = 0; typedef struct on_connect_result { + /* Owns a ref to server. */ + grpc_tcp_server *server; unsigned port_index; unsigned fd_index; int server_fd; } on_connect_result; -static on_connect_result g_result = {0, 0, -1}; +void on_connect_result_init(on_connect_result *result) { + result->server = NULL; + result->port_index = 0; + result->fd_index = 0; + result->server_fd = -1; +} + +void on_connect_result_set(on_connect_result *result, + const grpc_tcp_server_acceptor *acceptor) { + result->server = grpc_tcp_server_ref(acceptor->from_server); + result->port_index = acceptor->port_index; + result->fd_index = acceptor->fd_index; + result->server_fd = grpc_tcp_server_port_fd( + result->server, acceptor->port_index, acceptor->fd_index); +} + +static on_connect_result g_result = {NULL, 0, 0, -1}; static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_tcp_server_acceptor *acceptor) { @@ -65,10 +84,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); - g_result.port_index = acceptor->port_index; - g_result.fd_index = acceptor->fd_index; - g_result.server_fd = grpc_tcp_server_port_fd( - acceptor->from_server, acceptor->port_index, acceptor->fd_index); + on_connect_result_set(&g_result, acceptor); g_nconnects++; grpc_pollset_kick(&g_pollset, NULL); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); @@ -130,7 +146,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); nconnects_before = g_nconnects; - g_result.server_fd = -1; + on_connect_result_init(&g_result); GPR_ASSERT(clifd >= 0); gpr_log(GPR_DEBUG, "start connect"); GPR_ASSERT(connect(clifd, remote, remote_len) == 0); @@ -154,7 +170,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, /* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for the same port should be tested. */ -static void test_connect(int svr1_port, unsigned n) { +static void test_connect(unsigned n) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; struct sockaddr_storage addr; struct sockaddr_storage addr1; @@ -162,6 +178,7 @@ static void test_connect(int svr1_port, unsigned n) { unsigned svr_fd_count; int svr_port; unsigned svr1_fd_count; + int svr1_port; grpc_tcp_server *s = grpc_tcp_server_create(NULL); grpc_pollset *pollsets[1]; unsigned i; @@ -172,6 +189,9 @@ static void test_connect(int svr1_port, unsigned n) { addr.ss_family = addr1.ss_family = AF_INET; svr_port = grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len); GPR_ASSERT(svr_port > 0); + /* Cannot use wildcard (port==0), because add_port() will try to reuse the + same port as a previous add_port(). */ + svr1_port = grpc_pick_unused_port_or_die(); grpc_sockaddr_set_port((struct sockaddr *)&addr1, svr1_port); GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr1, addr_len) == svr1_port); @@ -211,8 +231,9 @@ static void test_connect(int svr1_port, unsigned n) { grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL); for (i = 0; i < n; i++) { - on_connect_result result = {0, 0, -1}; + on_connect_result result; int svr_fd; + on_connect_result_init(&result); tcp_connect(&exec_ctx, (struct sockaddr *)&addr, addr_len, &result); GPR_ASSERT(result.server_fd >= 0); svr_fd = result.server_fd; @@ -220,10 +241,10 @@ static void test_connect(int svr1_port, unsigned n) { result.server_fd); GPR_ASSERT(result.port_index == 0); GPR_ASSERT(result.fd_index < svr_fd_count); + GPR_ASSERT(result.server == s); + grpc_tcp_server_unref(&exec_ctx, result.server); - result.port_index = 0; - result.fd_index = 0; - result.server_fd = -1; + on_connect_result_init(&result); tcp_connect(&exec_ctx, (struct sockaddr *)&addr1, addr_len, &result); GPR_ASSERT(result.server_fd >= 0); GPR_ASSERT(result.server_fd != svr_fd); @@ -231,6 +252,8 @@ static void test_connect(int svr1_port, unsigned n) { result.server_fd); GPR_ASSERT(result.port_index == 1); GPR_ASSERT(result.fd_index < svr_fd_count); + GPR_ASSERT(result.server == s); + grpc_tcp_server_unref(&exec_ctx, result.server); } grpc_tcp_server_unref(&exec_ctx, s); @@ -244,21 +267,20 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) { int main(int argc, char **argv) { grpc_closure destroyed; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - int svr1_port = grpc_pick_unused_port_or_die(); grpc_test_init(argc, argv); - grpc_iomgr_init(); + grpc_init(); grpc_pollset_init(&g_pollset); test_no_op(); test_no_op_with_start(); test_no_op_with_port(); test_no_op_with_port_and_start(); - test_connect(svr1_port, 1); - test_connect(svr1_port, 10); + test_connect(1); + test_connect(10); grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); - grpc_iomgr_shutdown(); + grpc_shutdown(); return 0; } From fc999adcc7570ef19e16bf580199babb98096d4c Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Thu, 14 Jan 2016 17:05:46 -0800 Subject: [PATCH 13/42] new iterator interface --- include/grpc/census.h | 45 ++++++-- src/core/census/tag_set.c | 187 +++++++++++++++++--------------- test/core/census/tag_set_test.c | 77 ++++++------- 3 files changed, 167 insertions(+), 142 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 2a62f4daca5..1d7e65cdd9d 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -354,8 +354,21 @@ typedef struct { #define CENSUS_TAG_IS_STATS(flags) (flags & CENSUS_TAG_STATS) #define CENSUS_TAG_IS_BINARY(flags) (flags & CENSUS_TAG_BINARY) -#define CENSUS_MAX_TAG_KV_LEN 255 /* Maximum length of key/value in a tag. */ -#define CENSUS_MAX_TAGS 255 /* Maximum number of tags in a tag set. */ +/* Maximum length of key/value in a tag. */ +#define CENSUS_MAX_TAG_KV_LEN 255 +/* Maximum number of propagatable tags. */ +#define CENSUS_MAX_PROPAGATED_TAGS 255 + +typedef struct { + int n_propagated_tags; /* number of propagated printable tags */ + int n_propagated_binary_tags; /* number of propagated binary tags */ + int n_local_tags; /* number of non-propagated (local) tags */ + int n_deleted_tags; /* number of tags that were deleted */ + int n_added_tags; /* number of tags that were added */ + int n_modified_tags; /* number of tags that were modified */ + int n_invalid_tags; /* number of tags with bad keys or values (e.g. + longer than CENSUS_MAX_TAG_KV_LEN) */ +} census_tag_set_create_stats; /* Create a new tag set, adding and removing tags from an existing tag set. @param base Base tag set to build upon. Can be NULL. @@ -365,9 +378,12 @@ typedef struct { both, but with NULL or zero-length values, will be deleted from the tag set. @param ntags number of tags in 'tags' + @param stats Information about the tag set created and actions taken during + its creation. */ census_tag_set *census_tag_set_create(const census_tag_set *base, - const census_tag *tags, int ntags); + const census_tag *tags, int ntags, + census_tag_set_create_stats *stats); /* Destroy a tag set created by census_tag_set_create(). Once this function has been called, the tag set cannot be reused. */ @@ -376,10 +392,25 @@ void census_tag_set_destroy(census_tag_set *tags); /* Get the number of tags in the tag set. */ int census_tag_set_ntags(const census_tag_set *tags); -/* Get a tag by it's index in the tag set. Returns 0 if the index is invalid - (<0 or >= census_tag_set_ntags). There is no guarantee on tag ordering. */ -int census_tag_set_get_tag_by_index(const census_tag_set *tags, int index, - census_tag *tag); +/* Structure used for tag set iteration. API clients should not use or + reference internal fields - neither their contents or presence/absence are + guaranteed. */ +typedef struct { + const census_tag_set *tags; + int base; + int index; + char *kvm; +} census_tag_set_iterator; + +/* Initialize a tag set iterator. Must be called before first use of the + iterator. */ +void census_tag_set_initialize_iterator(const census_tag_set *tags, + census_tag_set_iterator *iterator); + +/* Get the contents of the "next" tag in the tag set. If there are no more + tags in the tag set, returns 0 (and 'tag' contents will be unchanged), + otherwise returns 1. */ +int census_tag_set_next_tag(census_tag_set_iterator *iterator, census_tag *tag); /* Get a tag by its key. Returns 0 if the key is not present in the tag set. */ diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index e879ac06068..3b8efb19358 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -30,6 +30,13 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ +/* +- ability to add extra tags in encode? +- add drops error count to create_ts +- add mask to ntags? +- comment about key/value ptrs being to mem +- add comment about encode/decode being for RPC use only. +*/ #include #include @@ -112,11 +119,14 @@ struct raw_tag { // tag_set structs, one for each of the binary/printable propagated tags, and // one for everything else. struct census_tag_set { - struct tag_set propagated_tags; - struct tag_set propagated_binary_tags; - struct tag_set local_tags; + struct tag_set tags[3]; }; +// Indices into the tags member of census_tag_set +#define PROPAGATED_TAGS 0 +#define PROPAGATED_BINARY_TAGS 1 +#define LOCAL_TAGS 2 + // Extract a raw tag given a pointer (raw) to the tag header. Allow for some // extra bytes in the tag header (see encode/decode for usage: allows for // future expansion of the tag header). @@ -156,44 +166,19 @@ static bool tag_set_delete_tag(struct tag_set *tags, const char *key, return false; } -// Delete a tag from a tag set. -static void cts_delete_tag(census_tag_set *tags, const census_tag *tag, +// Delete a tag from a tag set, return true if it existed. +static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, size_t key_len) { - // use the to-be-deleted tag flags as a hint to determine the order in which - // we delete from the underlying tag sets. - if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { - if (CENSUS_TAG_IS_BINARY(tag->flags)) { - if (tag_set_delete_tag(&tags->propagated_binary_tags, tag->key, - key_len)) { - return; - } - if (tag_set_delete_tag(&tags->propagated_tags, tag->key, key_len)) return; - tag_set_delete_tag(&tags->local_tags, tag->key, key_len); - } else { - if (tag_set_delete_tag(&tags->propagated_tags, tag->key, key_len)) { - return; - } - if (tag_set_delete_tag(&tags->propagated_binary_tags, tag->key, key_len)) - return; - tag_set_delete_tag(&tags->local_tags, tag->key, key_len); - } - } else { - if (tag_set_delete_tag(&tags->local_tags, tag->key, key_len)) { - return; - } - if (tag_set_delete_tag(&tags->propagated_tags, tag->key, key_len)) return; - tag_set_delete_tag(&tags->propagated_binary_tags, tag->key, key_len); - } + return (tag_set_delete_tag(&tags->tags[LOCAL_TAGS], tag->key, key_len) || + tag_set_delete_tag(&tags->tags[PROPAGATED_TAGS], tag->key, key_len) || + tag_set_delete_tag(&tags->tags[PROPAGATED_BINARY_TAGS], tag->key, + key_len)); } // Add a tag to a tag set. static void tag_set_add_tag(struct tag_set *tags, const census_tag *tag, size_t key_len) { const size_t tag_size = key_len + tag->value_len + TAG_HEADER_SIZE; - // drop tag if too many. - if (tags->ntags == CENSUS_MAX_TAGS) { - return; - } if (tags->kvm_used + tag_size > tags->kvm_size) { // allocate new memory if needed tags->kvm_size += 2 * CENSUS_MAX_TAG_KV_LEN + TAG_HEADER_SIZE; @@ -224,12 +209,12 @@ static void cts_add_tag(census_tag_set *tags, const census_tag *tag, if (tag->value != NULL && tag->value_len != 0) { if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { if (CENSUS_TAG_IS_BINARY(tag->flags)) { - tag_set_add_tag(&tags->propagated_binary_tags, tag, key_len); + tag_set_add_tag(&tags->tags[PROPAGATED_BINARY_TAGS], tag, key_len); } else { - tag_set_add_tag(&tags->propagated_tags, tag, key_len); + tag_set_add_tag(&tags->tags[PROPAGATED_TAGS], tag, key_len); } } else { - tag_set_add_tag(&tags->local_tags, tag, key_len); + tag_set_add_tag(&tags->tags[LOCAL_TAGS], tag, key_len); } } } @@ -242,7 +227,7 @@ static void cts_add_tag(census_tag_set *tags, const census_tag *tag, // 4) if we are still looking for a not-deleted tag, then all the end portion // of the kvm is deleted. Just reduce the used amount of memory by the // appropriate amount. -static void tag_set_compress(struct tag_set *tags) { +static void tag_set_flatten(struct tag_set *tags) { if (tags->ntags == tags->ntags_alloc) return; bool find_deleted = true; // are we looking for deleted tags? char *kvp = tags->kvm; @@ -277,15 +262,17 @@ static void tag_set_compress(struct tag_set *tags) { } census_tag_set *census_tag_set_create(const census_tag_set *base, - const census_tag *tags, int ntags) { + const census_tag *tags, int ntags, + census_tag_set_create_stats *stats) { + int n_invalid_tags = 0; census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); if (base == NULL) { memset(new_ts, 0, sizeof(census_tag_set)); } else { - tag_set_copy(&new_ts->propagated_tags, &base->propagated_tags); - tag_set_copy(&new_ts->propagated_binary_tags, - &base->propagated_binary_tags); - tag_set_copy(&new_ts->local_tags, &base->local_tags); + tag_set_copy(&new_ts->tags[PROPAGATED_TAGS], &base->tags[PROPAGATED_TAGS]); + tag_set_copy(&new_ts->tags[PROPAGATED_BINARY_TAGS], + &base->tags[PROPAGATED_BINARY_TAGS]); + tag_set_copy(&new_ts->tags[LOCAL_TAGS], &base->tags[LOCAL_TAGS]); } for (int i = 0; i < ntags; i++) { const census_tag *tag = &tags[i]; @@ -294,61 +281,81 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN && tag->value_len <= CENSUS_MAX_TAG_KV_LEN) { cts_add_tag(new_ts, tag, key_len); + } else { + n_invalid_tags++; } } - tag_set_compress(&new_ts->propagated_tags); - tag_set_compress(&new_ts->propagated_binary_tags); - tag_set_compress(&new_ts->local_tags); + tag_set_flatten(&new_ts->tags[PROPAGATED_TAGS]); + tag_set_flatten(&new_ts->tags[PROPAGATED_BINARY_TAGS]); + tag_set_flatten(&new_ts->tags[LOCAL_TAGS]); + if (stats != NULL) { + stats->n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; + stats->n_propagated_binary_tags = + new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; + stats->n_local_tags = new_ts->tags[LOCAL_TAGS].ntags; + stats->n_invalid_tags = n_invalid_tags; + } return new_ts; } void census_tag_set_destroy(census_tag_set *tags) { - gpr_free(tags->propagated_tags.kvm); - gpr_free(tags->propagated_binary_tags.kvm); - gpr_free(tags->local_tags.kvm); + gpr_free(tags->tags[PROPAGATED_TAGS].kvm); + gpr_free(tags->tags[PROPAGATED_BINARY_TAGS].kvm); + gpr_free(tags->tags[LOCAL_TAGS].kvm); gpr_free(tags); } int census_tag_set_ntags(const census_tag_set *tags) { - return tags->propagated_tags.ntags + tags->propagated_binary_tags.ntags + - tags->local_tags.ntags; + return tags->tags[PROPAGATED_TAGS].ntags + + tags->tags[PROPAGATED_BINARY_TAGS].ntags + + tags->tags[LOCAL_TAGS].ntags; } -// Get the nth tag in a tag set. The caller must validate that index is -// in range. -static void tag_set_get_tag_by_index(const struct tag_set *tags, int index, - census_tag *tag) { - GPR_ASSERT(index < tags->ntags); - char *kvp = tags->kvm; - struct raw_tag raw; - kvp = decode_tag(&raw, kvp, 0); - for (int i = 0; i < index; i++) { - kvp = decode_tag(&raw, kvp, 0); +/* Initialize a tag set iterator. Must be called before first use of the + iterator. */ +void census_tag_set_initialize_iterator(const census_tag_set *tags, + census_tag_set_iterator *iterator) { + iterator->tags = tags; + iterator->index = 0; + if (tags->tags[PROPAGATED_TAGS].ntags != 0) { + iterator->base = PROPAGATED_TAGS; + iterator->kvm = tags->tags[PROPAGATED_TAGS].kvm; + } else if (tags->tags[PROPAGATED_BINARY_TAGS].ntags != 0) { + iterator->base = PROPAGATED_BINARY_TAGS; + iterator->kvm = tags->tags[PROPAGATED_BINARY_TAGS].kvm; + } else if (tags->tags[LOCAL_TAGS].ntags != 0) { + iterator->base = LOCAL_TAGS; + iterator->kvm = tags->tags[LOCAL_TAGS].kvm; + } else { + iterator->base = -1; + } +} + +/* Get the contents of the "next" tag in the tag set. If there are no more + tags in the tag set, returns 0 (and 'tag' contents will be unchanged), + otherwise returns 1. */ +int census_tag_set_next_tag(census_tag_set_iterator *iterator, + census_tag *tag) { + if (iterator->base < 0) { + return 0; } + struct raw_tag raw; + iterator->kvm = decode_tag(&raw, iterator->kvm, 0); tag->key = raw.key; tag->value = raw.value; tag->value_len = raw.value_len; tag->flags = raw.flags; -} - -int census_tag_set_get_tag_by_index(const census_tag_set *tags, int index, - census_tag *tag) { - if (index < 0) return 0; - if (index < tags->propagated_tags.ntags) { - tag_set_get_tag_by_index(&tags->propagated_tags, index, tag); - return 1; - } - index -= tags->propagated_tags.ntags; - if (index < tags->propagated_binary_tags.ntags) { - tag_set_get_tag_by_index(&tags->propagated_binary_tags, index, tag); - return 1; - } - index -= tags->propagated_binary_tags.ntags; - if (index < tags->local_tags.ntags) { - tag_set_get_tag_by_index(&tags->local_tags, index, tag); - return 1; + if (++iterator->index == iterator->tags->tags[iterator->base].ntags) { + do { + if (iterator->base == LOCAL_TAGS) { + iterator->base = -1; + return 1; + } + } while (iterator->tags->tags[++iterator->base].ntags == 0); + iterator->index = 0; + iterator->kvm = iterator->tags->tags[iterator->base].kvm; } - return 0; + return 1; } // Find a tag in a tag_set by key. Return true if found, false otherwise. @@ -375,10 +382,10 @@ int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, if (key_len == 1) { return 0; } - if (tag_set_get_tag_by_key(&tags->propagated_tags, key, key_len, tag) || - tag_set_get_tag_by_key(&tags->propagated_binary_tags, key, key_len, + if (tag_set_get_tag_by_key(&tags->tags[PROPAGATED_TAGS], key, key_len, tag) || + tag_set_get_tag_by_key(&tags->tags[PROPAGATED_BINARY_TAGS], key, key_len, tag) || - tag_set_get_tag_by_key(&tags->local_tags, key, key_len, tag)) { + tag_set_get_tag_by_key(&tags->tags[LOCAL_TAGS], key, key_len, tag)) { return 1; } return 0; @@ -422,12 +429,12 @@ static size_t tag_set_encode(const struct tag_set *tags, char *buffer, size_t census_tag_set_encode_propagated(const census_tag_set *tags, char *buffer, size_t buf_size) { - return tag_set_encode(&tags->propagated_tags, buffer, buf_size); + return tag_set_encode(&tags->tags[PROPAGATED_TAGS], buffer, buf_size); } size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, char *buffer, size_t buf_size) { - return tag_set_encode(&tags->propagated_binary_tags, buffer, buf_size); + return tag_set_encode(&tags->tags[PROPAGATED_BINARY_TAGS], buffer, buf_size); } // Decode a tag set. @@ -475,16 +482,16 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, census_tag_set *census_tag_set_decode(const char *buffer, size_t size, const char *bin_buffer, size_t bin_size) { census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); - memset(&new_ts->local_tags, 0, sizeof(struct tag_set)); + memset(&new_ts->tags[LOCAL_TAGS], 0, sizeof(struct tag_set)); if (buffer == NULL) { - memset(&new_ts->propagated_tags, 0, sizeof(struct tag_set)); + memset(&new_ts->tags[PROPAGATED_TAGS], 0, sizeof(struct tag_set)); } else { - tag_set_decode(&new_ts->propagated_tags, buffer, size); + tag_set_decode(&new_ts->tags[PROPAGATED_TAGS], buffer, size); } if (bin_buffer == NULL) { - memset(&new_ts->propagated_binary_tags, 0, sizeof(struct tag_set)); + memset(&new_ts->tags[PROPAGATED_BINARY_TAGS], 0, sizeof(struct tag_set)); } else { - tag_set_decode(&new_ts->propagated_binary_tags, bin_buffer, bin_size); + tag_set_decode(&new_ts->tags[PROPAGATED_BINARY_TAGS], bin_buffer, bin_size); } // TODO(aveitch): check that BINARY flag is correct for each type. return new_ts; diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index 4c286b3e4f7..140aa8117b7 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -108,46 +108,34 @@ static bool validate_tag(const census_tag_set *cts, const census_tag *tag) { // Create an empty tag_set. static void empty_test(void) { - struct census_tag_set *cts = census_tag_set_create(NULL, NULL, 0); + struct census_tag_set *cts = census_tag_set_create(NULL, NULL, 0, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_destroy(cts); } -// Create basic tag set, and test that retreiving tag by index works. +// Test create and iteration over basic tag set. static void basic_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - for (int i = 0; i < census_tag_set_ntags(cts); i++) { - census_tag tag; - GPR_ASSERT(census_tag_set_get_tag_by_index(cts, i, &tag) == 1); + census_tag_set_iterator it; + census_tag_set_initialize_iterator(cts, &it); + census_tag tag; + while (census_tag_set_next_tag(&it, &tag)) { // can't rely on tag return order: make sure it matches exactly one. int matches = 0; - for (int j = 0; j < BASIC_TAG_COUNT; j++) { - if (compare_tag(&tag, &basic_tags[j])) matches++; + for (int i = 0; i < BASIC_TAG_COUNT; i++) { + if (compare_tag(&tag, &basic_tags[i])) matches++; } GPR_ASSERT(matches == 1); } census_tag_set_destroy(cts); } -// Try census_tag_set_get_tag_by_index() with bad indices. -static void bad_index_test(void) { - struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag tag; - GPR_ASSERT(census_tag_set_get_tag_by_index(cts, -1, &tag) == 0); - GPR_ASSERT(census_tag_set_get_tag_by_index(cts, BASIC_TAG_COUNT, &tag) == 0); - GPR_ASSERT(census_tag_set_get_tag_by_index(cts, BASIC_TAG_COUNT + 1, &tag) == - 0); - census_tag_set_destroy(cts); -} - // Test that census_tag_set_get_tag_by_key(). static void lookup_by_key_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); census_tag tag; for (int i = 0; i < census_tag_set_ntags(cts); i++) { @@ -175,35 +163,35 @@ static void invalid_test(void) { // long keys, short value. Key lengths (including terminator) should be // <= 255 (CENSUS_MAX_TAG_KV_LEN) GPR_ASSERT(strlen(key) == 299); - struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1); + struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_destroy(cts); key[CENSUS_MAX_TAG_KV_LEN] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN); - cts = census_tag_set_create(NULL, &tag, 1); + cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_destroy(cts); key[CENSUS_MAX_TAG_KV_LEN - 1] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN - 1); - cts = census_tag_set_create(NULL, &tag, 1); + cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 1); census_tag_set_destroy(cts); // now try with long values tag.value_len = 300; - cts = census_tag_set_create(NULL, &tag, 1); + cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_destroy(cts); tag.value_len = CENSUS_MAX_TAG_KV_LEN + 1; - cts = census_tag_set_create(NULL, &tag, 1); + cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_destroy(cts); tag.value_len = CENSUS_MAX_TAG_KV_LEN; - cts = census_tag_set_create(NULL, &tag, 1); + cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 1); census_tag_set_destroy(cts); // 0 length key. key[0] = 0; - cts = census_tag_set_create(NULL, &tag, 1); + cts = census_tag_set_create(NULL, &tag, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_destroy(cts); } @@ -211,9 +199,9 @@ static void invalid_test(void) { // Make a copy of a tag set static void copy_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0); + struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); for (int i = 0; i < census_tag_set_ntags(cts2); i++) { census_tag tag; @@ -228,10 +216,10 @@ static void copy_test(void) { // replace a single tag value static void replace_value_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + REPLACE_VALUE_OFFSET, 1); + census_tag_set_create(cts, modify_tags + REPLACE_VALUE_OFFSET, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( @@ -244,10 +232,10 @@ static void replace_value_test(void) { // replace a single tags flags static void replace_flags_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1); + census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( @@ -260,10 +248,10 @@ static void replace_flags_test(void) { // delete a single tag. static void delete_tag_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1); + census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT - 1); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( @@ -275,10 +263,10 @@ static void delete_tag_test(void) { // add a single new tag. static void add_tag_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1); + census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT + 1); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( @@ -291,10 +279,10 @@ static void add_tag_test(void) { // test many changes at once. static void replace_add_delete_test(void) { struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT); + census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == 8); // validate tag set contents. Use specific indices into the two arrays // holding tag values. @@ -321,7 +309,7 @@ static void simple_encode_decode_test(void) { char buf1[1000]; char buf2[1000]; struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); GPR_ASSERT(census_tag_set_encode_propagated(cts, buf1, 1) == 0); size_t b1 = census_tag_set_encode_propagated(cts, buf1, 1000); @@ -352,10 +340,10 @@ static void complex_encode_decode_test(void) { char buf1[500]; char buf2[500]; struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT); + census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == 8); size_t b1 = census_tag_set_encode_propagated(cts2, buf1, 500); @@ -376,7 +364,6 @@ int main(int argc, char *argv[]) { grpc_test_init(argc, argv); empty_test(); basic_test(); - bad_index_test(); lookup_by_key_test(); invalid_test(); copy_test(); From 9c12bc252e7bb8cdfe051884862e17d4850f3ab0 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 13 Jan 2016 16:52:20 -0800 Subject: [PATCH 14/42] Add shutdown_starting callbacks to tcp_server. tcp_server_posix_test illustrates how this can be used to implement a weak referencing mechanism. --- src/core/iomgr/tcp_server.h | 7 ++++ src/core/iomgr/tcp_server_posix.c | 20 +++++++++- src/core/iomgr/tcp_server_windows.c | 20 +++++++++- test/core/iomgr/tcp_server_posix_test.c | 52 +++++++++++++++++++++++-- 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h index 5199c25eed5..8f3184ff1e0 100644 --- a/src/core/iomgr/tcp_server.h +++ b/src/core/iomgr/tcp_server.h @@ -34,6 +34,7 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H #define GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H +#include "src/core/iomgr/closure.h" #include "src/core/iomgr/endpoint.h" /* Forward decl of grpc_tcp_server */ @@ -90,6 +91,12 @@ int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index, /* Ref s and return s. */ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s); +/* shutdown_starting is called when ref count has reached zero and the server is + about to be destroyed. The server will be deleted after it returns. Calling + grpc_tcp_server_ref() from it has no effect. */ +void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s, + grpc_closure *shutdown_starting); + /* If the recount drops to zero, delete s, and call (exec_ctx==NULL) or enqueue a call (exec_ctx!=NULL) to shutdown_complete. */ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s); diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index 6a22c08edba..adf14aeb597 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -128,6 +128,9 @@ struct grpc_tcp_server { grpc_tcp_listener *tail; unsigned nports; + /* List of closures passed to shutdown_starting_add(). */ + grpc_closure_list shutdown_starting; + /* shutdown callback */ grpc_closure *shutdown_complete; @@ -144,6 +147,8 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) { s->active_ports = 0; s->destroyed_ports = 0; s->shutdown = 0; + s->shutdown_starting.head = NULL; + s->shutdown_starting.tail = NULL; s->shutdown_complete = shutdown_complete; s->on_accept_cb = NULL; s->on_accept_cb_arg = NULL; @@ -585,13 +590,26 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) { return s; } +void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s, + grpc_closure *shutdown_starting) { + gpr_mu_lock(&s->mu); + grpc_closure_list_add(&s->shutdown_starting, shutdown_starting, 1); + gpr_mu_unlock(&s->mu); +} + void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { if (gpr_unref(&s->refs)) { + /* Complete shutdown_starting work before destroying. */ + grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_lock(&s->mu); + grpc_exec_ctx_enqueue_list(&local_exec_ctx, &s->shutdown_starting); + gpr_mu_unlock(&s->mu); if (exec_ctx == NULL) { - grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_exec_ctx_flush(&local_exec_ctx); tcp_server_destroy(&local_exec_ctx, s); grpc_exec_ctx_finish(&local_exec_ctx); } else { + grpc_exec_ctx_finish(&local_exec_ctx); tcp_server_destroy(exec_ctx, s); } } diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index 46617a6c652..8ee8149f251 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -93,6 +93,9 @@ struct grpc_tcp_server { grpc_tcp_listener *head; grpc_tcp_listener *tail; + /* List of closures passed to shutdown_starting_add(). */ + grpc_closure_list shutdown_starting; + /* shutdown callback */ grpc_closure *shutdown_complete; }; @@ -108,6 +111,8 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) { s->on_accept_cb_arg = NULL; s->head = NULL; s->tail = NULL; + s->shutdown_starting.head = NULL; + s->shutdown_starting.tail = NULL; s->shutdown_complete = shutdown_complete; return s; } @@ -135,6 +140,13 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) { return s; } +void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s, + grpc_closure *shutdown_starting) { + gpr_mu_lock(&s->mu); + grpc_closure_list_add(&s->shutdown_starting, shutdown_starting, 1); + gpr_mu_unlock(&s->mu); +} + static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { int immediately_done = 0; grpc_tcp_listener *sp; @@ -158,11 +170,17 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { if (gpr_unref(&s->refs)) { + /* Complete shutdown_starting work before destroying. */ + grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_lock(&s->mu); + grpc_exec_ctx_enqueue_list(&local_exec_ctx, &s->shutdown_starting); + gpr_mu_unlock(&s->mu); if (exec_ctx == NULL) { - grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_exec_ctx_flush(&local_exec_ctx); tcp_server_destroy(&local_exec_ctx, s); grpc_exec_ctx_finish(&local_exec_ctx); } else { + grpc_exec_ctx_finish(&local_exec_ctx); tcp_server_destroy(exec_ctx, s); } } diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 0f92166ab26..23b368df3af 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -60,15 +60,24 @@ typedef struct on_connect_result { int server_fd; } on_connect_result; -void on_connect_result_init(on_connect_result *result) { +typedef struct server_weak_ref { + grpc_tcp_server *server; + + /* arg is this server_weak_ref. */ + grpc_closure server_shutdown; +} server_weak_ref; + +static on_connect_result g_result = {NULL, 0, 0, -1}; + +static void on_connect_result_init(on_connect_result *result) { result->server = NULL; result->port_index = 0; result->fd_index = 0; result->server_fd = -1; } -void on_connect_result_set(on_connect_result *result, - const grpc_tcp_server_acceptor *acceptor) { +static void on_connect_result_set(on_connect_result *result, + const grpc_tcp_server_acceptor *acceptor) { result->server = grpc_tcp_server_ref(acceptor->from_server); result->port_index = acceptor->port_index; result->fd_index = acceptor->fd_index; @@ -76,7 +85,29 @@ void on_connect_result_set(on_connect_result *result, result->server, acceptor->port_index, acceptor->fd_index); } -static on_connect_result g_result = {NULL, 0, 0, -1}; + +static void server_weak_ref_shutdown(grpc_exec_ctx *exec_ctx, void *arg, + int success) { + server_weak_ref *weak_ref = arg; + weak_ref->server = NULL; +} + +static void server_weak_ref_init(server_weak_ref *weak_ref) { + weak_ref->server = NULL; + grpc_closure_init(&weak_ref->server_shutdown, server_weak_ref_shutdown, + weak_ref); +} + +/* Make weak_ref->server_shutdown a shutdown_starting cb on server. + grpc_tcp_server promises that the server object will live until + weak_ref->server_shutdown has returned. A strong ref on grpc_tcp_server + should be held until server_weak_ref_set() returns to avoid a race where the + server is deleted before the shutdown_starting cb is added. */ +static void server_weak_ref_set(server_weak_ref *weak_ref, + grpc_tcp_server *server) { + grpc_tcp_server_shutdown_starting_add(server, &weak_ref->server_shutdown); + weak_ref->server = server; +} static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_tcp_server_acceptor *acceptor) { @@ -182,6 +213,8 @@ static void test_connect(unsigned n) { grpc_tcp_server *s = grpc_tcp_server_create(NULL); grpc_pollset *pollsets[1]; unsigned i; + server_weak_ref weak_ref; + server_weak_ref_init(&weak_ref); LOG_TEST("test_connect"); gpr_log(GPR_INFO, "clients=%d", n); memset(&addr, 0, sizeof(addr)); @@ -242,6 +275,9 @@ static void test_connect(unsigned n) { GPR_ASSERT(result.port_index == 0); GPR_ASSERT(result.fd_index < svr_fd_count); GPR_ASSERT(result.server == s); + if (weak_ref.server == NULL) { + server_weak_ref_set(&weak_ref, result.server); + } grpc_tcp_server_unref(&exec_ctx, result.server); on_connect_result_init(&result); @@ -256,7 +292,15 @@ static void test_connect(unsigned n) { grpc_tcp_server_unref(&exec_ctx, result.server); } + /* Weak ref to server valid until final unref. */ + GPR_ASSERT(weak_ref.server != NULL); + GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 0) >= 0); + grpc_tcp_server_unref(&exec_ctx, s); + + /* Weak ref lost. */ + GPR_ASSERT(weak_ref.server == NULL); + grpc_exec_ctx_finish(&exec_ctx); } From a88a22166d768fbfc2da14d88fada058cfad7735 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 15 Jan 2016 12:13:36 -0800 Subject: [PATCH 15/42] add creation status --- include/grpc/census.h | 22 ++++--- src/core/census/tag_set.c | 72 +++++++++++++++------ test/core/census/tag_set_test.c | 107 +++++++++++++++++--------------- 3 files changed, 125 insertions(+), 76 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 1d7e65cdd9d..04b2c9db5fb 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -368,22 +368,27 @@ typedef struct { int n_modified_tags; /* number of tags that were modified */ int n_invalid_tags; /* number of tags with bad keys or values (e.g. longer than CENSUS_MAX_TAG_KV_LEN) */ -} census_tag_set_create_stats; + int n_ignored_tags; /* number of tags ignored because of + CENSUS_MAX_PROPAGATED_TAGS limit. */ +} census_tag_set_create_status; /* Create a new tag set, adding and removing tags from an existing tag set. @param base Base tag set to build upon. Can be NULL. @param tags A set of tags to be added/changed/deleted. Tags with keys that are in 'tags', but not 'base', are added to the tag set. Keys that are in - both 'tags' and 'base' will have their value replaced. Tags with keys in - both, but with NULL or zero-length values, will be deleted from the - tag set. + both 'tags' and 'base' will have their value/flags modified. Tags with keys + in both, but with NULL or zero-length values, will be deleted from the tag + set. Tags with invalid (too long or short) keys or values will be ignored. + If adding a tag will result in more than CENSUS_MAX_PROPAGATED_TAGS in either + binary or non-binary tags, they will be ignored, as will deletions of + tags that don't exist. @param ntags number of tags in 'tags' @param stats Information about the tag set created and actions taken during its creation. */ census_tag_set *census_tag_set_create(const census_tag_set *base, const census_tag *tags, int ntags, - census_tag_set_create_stats *stats); + census_tag_set_create_status *status); /* Destroy a tag set created by census_tag_set_create(). Once this function has been called, the tag set cannot be reused. */ @@ -429,9 +434,12 @@ size_t census_tag_set_encode_propagated(const census_tag_set *tags, size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, char *buffer, size_t buf_size); -/* Decode tag set buffers encoded with census_tag_set_encode_*(). */ +/* Decode tag set buffers encoded with census_tag_set_encode_*(). Returns NULL + if there is an error in parsing either buffer. The number of tags in the + decoded tag set will be returned in status, if it is non-NULL. */ census_tag_set *census_tag_set_decode(const char *buffer, size_t size, - const char *bin_buffer, size_t bin_size); + const char *bin_buffer, size_t bin_size, + census_tag_set_create_status *status); /* Get a contexts tag set. */ census_tag_set *census_context_tag_set(census_context *context); diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 3b8efb19358..f5f32f5e0fb 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -31,9 +31,6 @@ * */ /* -- ability to add extra tags in encode? -- add drops error count to create_ts -- add mask to ntags? - comment about key/value ptrs being to mem - add comment about encode/decode being for RPC use only. */ @@ -175,9 +172,13 @@ static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, key_len)); } -// Add a tag to a tag set. -static void tag_set_add_tag(struct tag_set *tags, const census_tag *tag, +// Add a tag to a tag set. Return true on sucess, false if the tag could +// not be added because of tag size constraints. +static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, size_t key_len) { + if (tags->ntags == CENSUS_MAX_PROPAGATED_TAGS) { + return false; + } const size_t tag_size = key_len + tag->value_len + TAG_HEADER_SIZE; if (tags->kvm_used + tag_size > tags->kvm_size) { // allocate new memory if needed @@ -199,22 +200,41 @@ static void tag_set_add_tag(struct tag_set *tags, const census_tag *tag, tags->kvm_used += tag_size; tags->ntags++; tags->ntags_alloc++; + return true; } -// Add a tag to a census_tag_set +// Add a tag to a census_tag_set. static void cts_add_tag(census_tag_set *tags, const census_tag *tag, - size_t key_len) { + size_t key_len, census_tag_set_create_status *status) { // first delete the tag if it is already present - cts_delete_tag(tags, tag, key_len); - if (tag->value != NULL && tag->value_len != 0) { + bool deleted = cts_delete_tag(tags, tag, key_len); + bool call_add = tag->value != NULL && tag->value_len != 0; + bool added = false; + if (call_add) { if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { if (CENSUS_TAG_IS_BINARY(tag->flags)) { - tag_set_add_tag(&tags->tags[PROPAGATED_BINARY_TAGS], tag, key_len); + added = + tag_set_add_tag(&tags->tags[PROPAGATED_BINARY_TAGS], tag, key_len); + } else { + added = tag_set_add_tag(&tags->tags[PROPAGATED_TAGS], tag, key_len); + } + } else { + added = tag_set_add_tag(&tags->tags[LOCAL_TAGS], tag, key_len); + } + } + if (status) { + if (deleted) { + if (call_add) { + status->n_modified_tags++; } else { - tag_set_add_tag(&tags->tags[PROPAGATED_TAGS], tag, key_len); + status->n_deleted_tags++; } } else { - tag_set_add_tag(&tags->tags[LOCAL_TAGS], tag, key_len); + if (added) { + status->n_added_tags++; + } else { + status->n_ignored_tags++; + } } } } @@ -263,8 +283,11 @@ static void tag_set_flatten(struct tag_set *tags) { census_tag_set *census_tag_set_create(const census_tag_set *base, const census_tag *tags, int ntags, - census_tag_set_create_stats *stats) { + census_tag_set_create_status *status) { int n_invalid_tags = 0; + if (status) { + memset(status, 0, sizeof(*status)); + } census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); if (base == NULL) { memset(new_ts, 0, sizeof(census_tag_set)); @@ -280,7 +303,7 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, // ignore the tag if it is too long/short. if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN && tag->value_len <= CENSUS_MAX_TAG_KV_LEN) { - cts_add_tag(new_ts, tag, key_len); + cts_add_tag(new_ts, tag, key_len, status); } else { n_invalid_tags++; } @@ -288,12 +311,12 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, tag_set_flatten(&new_ts->tags[PROPAGATED_TAGS]); tag_set_flatten(&new_ts->tags[PROPAGATED_BINARY_TAGS]); tag_set_flatten(&new_ts->tags[LOCAL_TAGS]); - if (stats != NULL) { - stats->n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; - stats->n_propagated_binary_tags = + if (status != NULL) { + status->n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; + status->n_propagated_binary_tags = new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; - stats->n_local_tags = new_ts->tags[LOCAL_TAGS].ntags; - stats->n_invalid_tags = n_invalid_tags; + status->n_local_tags = new_ts->tags[LOCAL_TAGS].ntags; + status->n_invalid_tags = n_invalid_tags; } return new_ts; } @@ -480,7 +503,11 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, } census_tag_set *census_tag_set_decode(const char *buffer, size_t size, - const char *bin_buffer, size_t bin_size) { + const char *bin_buffer, size_t bin_size, + census_tag_set_create_status *status) { + if (status) { + memset(status, 0, sizeof(*status)); + } census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); memset(&new_ts->tags[LOCAL_TAGS], 0, sizeof(struct tag_set)); if (buffer == NULL) { @@ -493,6 +520,11 @@ census_tag_set *census_tag_set_decode(const char *buffer, size_t size, } else { tag_set_decode(&new_ts->tags[PROPAGATED_BINARY_TAGS], bin_buffer, bin_size); } + if (status) { + status->n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; + status->n_propagated_binary_tags = + new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; + } // TODO(aveitch): check that BINARY flag is correct for each type. return new_ts; } diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index 140aa8117b7..bc084ec04bd 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -70,7 +70,7 @@ static census_tag basic_tags[BASIC_TAG_COUNT] = { // you add or delete entries, you will also need to change the test. Other // tests that rely on specific instances have XXX_XXX_OFFSET definitions (also // change the defines below if you add/delete entires). -#define MODIFY_TAG_COUNT 10 +#define MODIFY_TAG_COUNT 11 static census_tag modify_tags[MODIFY_TAG_COUNT] = { #define REPLACE_VALUE_OFFSET 0 /* 0 */ {"key0", "replace printable", 18, 0}, // replaces tag value only @@ -88,8 +88,8 @@ static census_tag modify_tags[MODIFY_TAG_COUNT] = { /* 8 */ {"k2", (char *)&eight_byte_val, 8, CENSUS_TAG_BINARY | CENSUS_TAG_PROPAGATE}, // more flags change // non-binary -> binary - /* 9 */ {"k6", "bar", 4, - 0} // add back tag, with different value, but same length + /* 9 */ {"k6", "bar", 4, 0}, // add back tag, with different value + /* 10 */ {"foo", "bar", 4, CENSUS_TAG_PROPAGATE}, // another new tag }; // Utility function to compare tags. Returns true if all fields match. @@ -115,9 +115,12 @@ static void empty_test(void) { // Test create and iteration over basic tag set. static void basic_test(void) { + census_tag_set_create_status status; struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); + census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, &status); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag_set_create_status expected = {2, 2, 4, 0, 8, 0, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag_set_iterator it; census_tag_set_initialize_iterator(cts, &it); census_tag tag; @@ -163,36 +166,46 @@ static void invalid_test(void) { // long keys, short value. Key lengths (including terminator) should be // <= 255 (CENSUS_MAX_TAG_KV_LEN) GPR_ASSERT(strlen(key) == 299); - struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1, NULL); + census_tag_set_create_status status; + struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 0); + census_tag_set_create_status expected = {0, 0, 0, 0, 0, 0, 1, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag_set_destroy(cts); key[CENSUS_MAX_TAG_KV_LEN] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN); - cts = census_tag_set_create(NULL, &tag, 1, NULL); + cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 0); + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag_set_destroy(cts); key[CENSUS_MAX_TAG_KV_LEN - 1] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN - 1); - cts = census_tag_set_create(NULL, &tag, 1, NULL); + cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 1); + census_tag_set_create_status expected2 = {0, 0, 1, 0, 1, 0, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected2, sizeof(status)) == 0); census_tag_set_destroy(cts); // now try with long values tag.value_len = 300; - cts = census_tag_set_create(NULL, &tag, 1, NULL); + cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 0); + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag_set_destroy(cts); tag.value_len = CENSUS_MAX_TAG_KV_LEN + 1; - cts = census_tag_set_create(NULL, &tag, 1, NULL); + cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 0); + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag_set_destroy(cts); tag.value_len = CENSUS_MAX_TAG_KV_LEN; - cts = census_tag_set_create(NULL, &tag, 1, NULL); + cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 1); + GPR_ASSERT(memcmp(&status, &expected2, sizeof(status)) == 0); census_tag_set_destroy(cts); // 0 length key. key[0] = 0; - cts = census_tag_set_create(NULL, &tag, 1, NULL); + cts = census_tag_set_create(NULL, &tag, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts) == 0); + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag_set_destroy(cts); } @@ -201,8 +214,11 @@ static void copy_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0, NULL); + census_tag_set_create_status status; + struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0, &status); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); + census_tag_set_create_status expected = {2, 2, 4, 0, 0, 0, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); for (int i = 0; i < census_tag_set_ntags(cts2); i++) { census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) == @@ -218,9 +234,12 @@ static void replace_value_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + REPLACE_VALUE_OFFSET, 1, NULL); + census_tag_set_create_status status; + struct census_tag_set *cts2 = census_tag_set_create( + cts, modify_tags + REPLACE_VALUE_OFFSET, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); + census_tag_set_create_status expected = {2, 2, 4, 0, 0, 1, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[REPLACE_VALUE_OFFSET].key, &tag) == 1); @@ -234,9 +253,12 @@ static void replace_flags_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag_set_create_status status; struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1, NULL); + census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); + census_tag_set_create_status expected = {1, 2, 5, 0, 0, 1, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[REPLACE_FLAG_OFFSET].key, &tag) == 1); @@ -250,9 +272,12 @@ static void delete_tag_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag_set_create_status status; struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1, NULL); + census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT - 1); + census_tag_set_create_status expected = {2, 1, 4, 1, 0, 0, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[DELETE_TAG_OFFSET].key, &tag) == 0); @@ -265,9 +290,12 @@ static void add_tag_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag_set_create_status status; struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1, NULL); + census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1, &status); GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT + 1); + census_tag_set_create_status expected = {2, 2, 5, 0, 1, 0, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[ADD_TAG_OFFSET].key, &tag) == 1); @@ -281,9 +309,12 @@ static void replace_add_delete_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); + census_tag_set_create_status status; struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts2) == 8); + census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, &status); + GPR_ASSERT(census_tag_set_ntags(cts2) == 9); + census_tag_set_create_status expected = {2, 1, 6, 2, 3, 4, 0, 2}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); // validate tag set contents. Use specific indices into the two arrays // holding tag values. GPR_ASSERT(validate_tag(cts2, &basic_tags[3])); @@ -294,6 +325,7 @@ static void replace_add_delete_test(void) { GPR_ASSERT(validate_tag(cts2, &modify_tags[7])); GPR_ASSERT(validate_tag(cts2, &modify_tags[8])); GPR_ASSERT(validate_tag(cts2, &modify_tags[9])); + GPR_ASSERT(validate_tag(cts2, &modify_tags[10])); GPR_ASSERT(!validate_tag(cts2, &basic_tags[0])); GPR_ASSERT(!validate_tag(cts2, &basic_tags[1])); GPR_ASSERT(!validate_tag(cts2, &basic_tags[2])); @@ -304,8 +336,8 @@ static void replace_add_delete_test(void) { census_tag_set_destroy(cts2); } -// Use the basic tag set to test encode/decode. -static void simple_encode_decode_test(void) { +// test encode/decode. +static void encode_decode_test(void) { char buf1[1000]; char buf2[1000]; struct census_tag_set *cts = @@ -317,9 +349,12 @@ static void simple_encode_decode_test(void) { GPR_ASSERT(census_tag_set_encode_propagated_binary(cts, buf2, 1) == 0); size_t b2 = census_tag_set_encode_propagated_binary(cts, buf2, 1000); GPR_ASSERT(b2 != 0); - census_tag_set *cts2 = census_tag_set_decode(buf1, b1, buf2, b2); + census_tag_set_create_status status; + census_tag_set *cts2 = census_tag_set_decode(buf1, b1, buf2, b2, &status); GPR_ASSERT(cts2 != NULL); GPR_ASSERT(census_tag_set_ntags(cts2) == 4); + census_tag_set_create_status expected = {2, 2, 0, 0, 0, 0, 0, 0}; + GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); for (int i = 0; i < census_tag_set_ntags(cts); i++) { census_tag tag; if (CENSUS_TAG_IS_PROPAGATED(basic_tags[i].flags)) { @@ -335,31 +370,6 @@ static void simple_encode_decode_test(void) { census_tag_set_destroy(cts); } -// Use more complex/modified tag set to test encode/decode. -static void complex_encode_decode_test(void) { - char buf1[500]; - char buf2[500]; - struct census_tag_set *cts = - census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - struct census_tag_set *cts2 = - census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts2) == 8); - - size_t b1 = census_tag_set_encode_propagated(cts2, buf1, 500); - GPR_ASSERT(b1 != 0); - size_t b2 = census_tag_set_encode_propagated_binary(cts2, buf2, 500); - GPR_ASSERT(b2 != 0); - census_tag_set *cts3 = census_tag_set_decode(buf1, b1, buf2, b2); - GPR_ASSERT(cts3 != NULL); - GPR_ASSERT(census_tag_set_ntags(cts3) == 2); - GPR_ASSERT(validate_tag(cts3, &basic_tags[4])); - GPR_ASSERT(validate_tag(cts3, &modify_tags[8])); - census_tag_set_destroy(cts3); - census_tag_set_destroy(cts2); - census_tag_set_destroy(cts); -} - int main(int argc, char *argv[]) { grpc_test_init(argc, argv); empty_test(); @@ -372,7 +382,6 @@ int main(int argc, char *argv[]) { delete_tag_test(); add_tag_test(); replace_add_delete_test(); - simple_encode_decode_test(); - complex_encode_decode_test(); + encode_decode_test(); return 0; } From 04de8c16d2f1fc4f9c97e460ea1f450872981d58 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 15 Jan 2016 13:15:03 -0800 Subject: [PATCH 16/42] add comments --- include/grpc/census.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 04b2c9db5fb..7b769c62eeb 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -329,7 +329,7 @@ void census_trace_scan_end(); a tag set. All contexts have an associated tag set. */ typedef struct census_tag_set census_tag_set; -/* A tag is a key:value pair. The key is a printable, nil-terminate +/* A tag is a key:value pair. The key is a non-empty, printable, nil-terminated string. The value is a binary string, that may be printable. There are no limits on the sizes of either keys or values, but code authors should remember that systems may have inbuilt limits (e.g. for propagated tags, @@ -422,6 +422,10 @@ int census_tag_set_next_tag(census_tag_set_iterator *iterator, census_tag *tag); int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, census_tag *tag); +/* Tag set encode/decode functionality. These functionas are intended + for use by RPC systems only, for purposes of transmitting/receiving tag + sets. */ + /* Encode to-be-propagated non-binary tags from a tag set into a memory buffer. The total number of bytes used in the buffer is returned. If the buffer is too small to contain the encoded tag set, then 0 is returned. */ From 36b24109a42178d894b8c80f005bcb551c4986fb Mon Sep 17 00:00:00 2001 From: Dan Born Date: Fri, 15 Jan 2016 17:52:58 -0800 Subject: [PATCH 17/42] clang reformat --- test/core/iomgr/tcp_server_posix_test.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 23b368df3af..f7097ac904d 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -85,7 +85,6 @@ static void on_connect_result_set(on_connect_result *result, result->server, acceptor->port_index, acceptor->fd_index); } - static void server_weak_ref_shutdown(grpc_exec_ctx *exec_ctx, void *arg, int success) { server_weak_ref *weak_ref = arg; From 4bbdb825677bde8e53cbab58ad4193c505dc9a50 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Tue, 19 Jan 2016 13:56:52 -0800 Subject: [PATCH 18/42] comment updates --- include/grpc/census.h | 24 +++++++++-------- src/core/census/tag_set.c | 56 +++++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 7b769c62eeb..4b3d0214104 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -329,12 +329,14 @@ void census_trace_scan_end(); a tag set. All contexts have an associated tag set. */ typedef struct census_tag_set census_tag_set; -/* A tag is a key:value pair. The key is a non-empty, printable, nil-terminated - string. The value is a binary string, that may be printable. There are no - limits on the sizes of either keys or values, but code authors should - remember that systems may have inbuilt limits (e.g. for propagated tags, - the bytes on the wire) and that larger tags means more memory consumed and - time in processing. */ +/* A tag is a key:value pair. The key is a non-empty, printable (UTF-8 + encoded), nil-terminated string. The value is a binary string, that may be + printable. There are limits on the sizes of both keys and values (see + CENSUS_MAX_TAG_KB_LEN definition below), and the number of tags that can be + propagated (CENSUS_MAX_PROPAGATED_TAGS). Users should also remember that + some systems may have limits on, e.g., the number of bytes that can be + transmitted as metadata, and that larger tags means more memory consumed + and time in processing. */ typedef struct { const char *key; const char *value; @@ -342,6 +344,11 @@ typedef struct { uint8_t flags; } census_tag; +/* Maximum length of a tag's key or value. */ +#define CENSUS_MAX_TAG_KV_LEN 255 +/* Maximum number of propagatable tags. */ +#define CENSUS_MAX_PROPAGATED_TAGS 255 + /* Tag flags. */ #define CENSUS_TAG_PROPAGATE 1 /* Tag should be propagated over RPC */ #define CENSUS_TAG_STATS 2 /* Tag will be used for statistics aggregation */ @@ -354,11 +361,6 @@ typedef struct { #define CENSUS_TAG_IS_STATS(flags) (flags & CENSUS_TAG_STATS) #define CENSUS_TAG_IS_BINARY(flags) (flags & CENSUS_TAG_BINARY) -/* Maximum length of key/value in a tag. */ -#define CENSUS_MAX_TAG_KV_LEN 255 -/* Maximum number of propagatable tags. */ -#define CENSUS_MAX_PROPAGATED_TAGS 255 - typedef struct { int n_propagated_tags; /* number of propagated printable tags */ int n_propagated_binary_tags; /* number of propagated binary tags */ diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index f5f32f5e0fb..718cfe3b253 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -30,10 +30,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ -/* -- comment about key/value ptrs being to mem -- add comment about encode/decode being for RPC use only. -*/ #include #include @@ -65,10 +61,10 @@ // that can be directly copied to the wire. This makes iteration by index // somewhat less efficient. If it becomes a problem, we could consider // building an index at tag_set creation. -// * Binary tags are encoded seperately from non-binary tags. There are a -// primarily because non-binary tags are far more likely to be repeated -// across multiple RPC calls, so are more efficiently cached and -// compressed in any metadata schemes. +// * Binary tags share the same structure as, but are encoded seperately from, +// non-binary tags. This is primarily because non-binary tags are far more +// likely to be repeated across multiple RPC calls, so are more efficiently +// cached and compressed in any metadata schemes. // * all lengths etc. are restricted to one byte. This eliminates endian // issues. @@ -108,7 +104,7 @@ struct raw_tag { char *value; }; -// use reserved flag bit for indication of deleted tag. +// Use a reserved flag bit for indication of deleted tag. #define CENSUS_TAG_DELETED CENSUS_TAG_RESERVED #define CENSUS_TAG_IS_DELETED(flags) (flags & CENSUS_TAG_DELETED) @@ -125,8 +121,8 @@ struct census_tag_set { #define LOCAL_TAGS 2 // Extract a raw tag given a pointer (raw) to the tag header. Allow for some -// extra bytes in the tag header (see encode/decode for usage: allows for -// future expansion of the tag header). +// extra bytes in the tag header (see encode/decode functions for usage: this +// allows for future expansion of the tag header). static char *decode_tag(struct raw_tag *tag, char *header, int offset) { tag->key_len = (uint8_t)(*header++); tag->value_len = (uint8_t)(*header++); @@ -145,7 +141,7 @@ static void tag_set_copy(struct tag_set *to, const struct tag_set *from) { memcpy(to->kvm, from->kvm, to->kvm_used); } -// Delete a tag from a tag set, if it exists (returns true it it did). +// Delete a tag from a tag_set, if it exists (returns true it it did). static bool tag_set_delete_tag(struct tag_set *tags, const char *key, size_t key_len) { char *kvp = tags->kvm; @@ -163,7 +159,7 @@ static bool tag_set_delete_tag(struct tag_set *tags, const char *key, return false; } -// Delete a tag from a tag set, return true if it existed. +// Delete a tag from a census_tag_set, return true if it existed. static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, size_t key_len) { return (tag_set_delete_tag(&tags->tags[LOCAL_TAGS], tag->key, key_len) || @@ -172,8 +168,8 @@ static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, key_len)); } -// Add a tag to a tag set. Return true on sucess, false if the tag could -// not be added because of tag size constraints. +// Add a tag to a tag_set. Return true on sucess, false if the tag could +// not be added because of constraints on tag set size. static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, size_t key_len) { if (tags->ntags == CENSUS_MAX_PROPAGATED_TAGS) { @@ -203,11 +199,14 @@ static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, return true; } -// Add a tag to a census_tag_set. -static void cts_add_tag(census_tag_set *tags, const census_tag *tag, - size_t key_len, census_tag_set_create_status *status) { - // first delete the tag if it is already present +// Add/modify/delete a tag to/in a census_tag_set. Caller must validate that +// tag key etc. are valid. +static void cts_modify_tag(census_tag_set *tags, const census_tag *tag, + size_t key_len, + census_tag_set_create_status *status) { + // First delete the tag if it is already present. bool deleted = cts_delete_tag(tags, tag, key_len); + // Determine if we need to add it back. bool call_add = tag->value != NULL && tag->value_len != 0; bool added = false; if (call_add) { @@ -239,7 +238,7 @@ static void cts_add_tag(census_tag_set *tags, const census_tag *tag, } } -// Remove any deleted tags from the tag set. Basic algorithm: +// Remove memory used for deleted tags from the tag set. Basic algorithm: // 1) Walk through tag set to find first deleted tag. Record where it is. // 2) Find the next not-deleted tag. Copy all of kvm from there to the end // "over" the deleted tags @@ -289,6 +288,8 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, memset(status, 0, sizeof(*status)); } census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); + // If we are given a base, copy it into our new tag set. Otherwise set it + // to zero/NULL everything. if (base == NULL) { memset(new_ts, 0, sizeof(census_tag_set)); } else { @@ -297,17 +298,20 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, &base->tags[PROPAGATED_BINARY_TAGS]); tag_set_copy(&new_ts->tags[LOCAL_TAGS], &base->tags[LOCAL_TAGS]); } + // Walk over the additional tags and, for those that aren't invalid, modify + // the tag set to add/replace/delete as required. for (int i = 0; i < ntags; i++) { const census_tag *tag = &tags[i]; size_t key_len = strlen(tag->key) + 1; // ignore the tag if it is too long/short. if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN && tag->value_len <= CENSUS_MAX_TAG_KV_LEN) { - cts_add_tag(new_ts, tag, key_len, status); + cts_modify_tag(new_ts, tag, key_len, status); } else { n_invalid_tags++; } } + // Remove any deleted tags, update status if needed, and return. tag_set_flatten(&new_ts->tags[PROPAGATED_TAGS]); tag_set_flatten(&new_ts->tags[PROPAGATED_BINARY_TAGS]); tag_set_flatten(&new_ts->tags[LOCAL_TAGS]); @@ -334,8 +338,8 @@ int census_tag_set_ntags(const census_tag_set *tags) { tags->tags[LOCAL_TAGS].ntags; } -/* Initialize a tag set iterator. Must be called before first use of the - iterator. */ +// Initialize a tag set iterator. Must be called before first use of the +// iterator. void census_tag_set_initialize_iterator(const census_tag_set *tags, census_tag_set_iterator *iterator) { iterator->tags = tags; @@ -354,9 +358,9 @@ void census_tag_set_initialize_iterator(const census_tag_set *tags, } } -/* Get the contents of the "next" tag in the tag set. If there are no more - tags in the tag set, returns 0 (and 'tag' contents will be unchanged), - otherwise returns 1. */ +// Get the contents of the "next" tag in the tag set. If there are no more +// tags in the tag set, returns 0 (and 'tag' contents will be unchanged), +// otherwise returns 1. */ int census_tag_set_next_tag(census_tag_set_iterator *iterator, census_tag *tag) { if (iterator->base < 0) { From 8e5b21fd9faf0b1ba2f74e88fe7e59bc3d5ed299 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Wed, 20 Jan 2016 09:17:11 -0800 Subject: [PATCH 19/42] comment updates --- include/grpc/census.h | 10 +++++++--- src/core/census/tag_set.c | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 4b3d0214104..5443488fbe4 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -375,6 +375,8 @@ typedef struct { } census_tag_set_create_status; /* Create a new tag set, adding and removing tags from an existing tag set. + This will copy all tags from it's input parameters, so it is recommended + to add as many tags in a single operation as is practical for the client. @param base Base tag set to build upon. Can be NULL. @param tags A set of tags to be added/changed/deleted. Tags with keys that are in 'tags', but not 'base', are added to the tag set. Keys that are in @@ -385,8 +387,10 @@ typedef struct { binary or non-binary tags, they will be ignored, as will deletions of tags that don't exist. @param ntags number of tags in 'tags' - @param stats Information about the tag set created and actions taken during - its creation. + @param status If not NULL, the pointed to structure will be filled in with + information about the new tag set and status of the tags used in its + creation. + @return A new, valid census_tag_set. */ census_tag_set *census_tag_set_create(const census_tag_set *base, const census_tag *tags, int ntags, @@ -396,7 +400,7 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, has been called, the tag set cannot be reused. */ void census_tag_set_destroy(census_tag_set *tags); -/* Get the number of tags in the tag set. */ +/* Get the total number of tags in the tag set. */ int census_tag_set_ntags(const census_tag_set *tags); /* Structure used for tag set iteration. API clients should not use or diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 718cfe3b253..3b5d3459102 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -56,7 +56,8 @@ // are to aid in efficient parsing and the ability to directly return key // strings. This is more important than saving a single byte/tag on the wire. // * The wire encoding uses only single byte values. This eliminates the need -// to handle endian-ness conversions. +// to handle endian-ness conversions. It also means there is a hard upper +// limit of 255 for both CENSUS_MAX_TAG_KV_LEN and CENSUS_MAX_PROPAGATED_TAGS. // * Keep all tag information (keys/values/flags) in a single memory buffer, // that can be directly copied to the wire. This makes iteration by index // somewhat less efficient. If it becomes a problem, we could consider From 9d8dabb5522bb877ee506a67ce370b649405338d Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Thu, 21 Jan 2016 13:34:41 -0800 Subject: [PATCH 20/42] address tag_set comments --- src/core/census/tag_set.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 3b5d3459102..5cc777b085f 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -41,7 +41,7 @@ #include "src/core/support/string.h" // Functions in this file support the public tag_set API, as well as -// encoding/decoding tag_sets as part of context transmission across +// encoding/decoding tag_sets as part of context propagation across // RPC's. The overall requirements (in approximate priority order) for the // tag_set representations: // 1. Efficient conversion to/from wire format @@ -59,10 +59,8 @@ // to handle endian-ness conversions. It also means there is a hard upper // limit of 255 for both CENSUS_MAX_TAG_KV_LEN and CENSUS_MAX_PROPAGATED_TAGS. // * Keep all tag information (keys/values/flags) in a single memory buffer, -// that can be directly copied to the wire. This makes iteration by index -// somewhat less efficient. If it becomes a problem, we could consider -// building an index at tag_set creation. -// * Binary tags share the same structure as, but are encoded seperately from, +// that can be directly copied to the wire.14 +// * Binary tags share the same structure as, but are encoded separately from, // non-binary tags. This is primarily because non-binary tags are far more // likely to be repeated across multiple RPC calls, so are more efficiently // cached and compressed in any metadata schemes. @@ -111,7 +109,8 @@ struct raw_tag { // Primary (external) representation of a tag set. Composed of 3 underlying // tag_set structs, one for each of the binary/printable propagated tags, and -// one for everything else. +// one for everything else. This is to efficiently support tag +// encoding/decoding. struct census_tag_set { struct tag_set tags[3]; }; @@ -139,10 +138,10 @@ static char *decode_tag(struct raw_tag *tag, char *header, int offset) { static void tag_set_copy(struct tag_set *to, const struct tag_set *from) { memcpy(to, from, sizeof(struct tag_set)); to->kvm = gpr_malloc(to->kvm_size); - memcpy(to->kvm, from->kvm, to->kvm_used); + memcpy(to->kvm, from->kvm, from->kvm_used); } -// Delete a tag from a tag_set, if it exists (returns true it it did). +// Delete a tag from a tag_set, if it exists (returns true if it did). static bool tag_set_delete_tag(struct tag_set *tags, const char *key, size_t key_len) { char *kvp = tags->kvm; @@ -249,18 +248,13 @@ static void cts_modify_tag(census_tag_set *tags, const census_tag *tag, // appropriate amount. static void tag_set_flatten(struct tag_set *tags) { if (tags->ntags == tags->ntags_alloc) return; - bool find_deleted = true; // are we looking for deleted tags? + bool found_deleted = false; // found a deleted tag. char *kvp = tags->kvm; char *dbase; // record location of deleted tag for (int i = 0; i < tags->ntags_alloc; i++) { struct raw_tag tag; char *next_kvp = decode_tag(&tag, kvp, 0); - if (find_deleted) { - if (CENSUS_TAG_IS_DELETED(tag.flags)) { - dbase = kvp; - find_deleted = false; - } - } else { + if (found_deleted) { if (!CENSUS_TAG_IS_DELETED(tag.flags)) { ptrdiff_t reduce = kvp - dbase; // #bytes in deleted tags GPR_ASSERT(reduce > 0); @@ -269,12 +263,17 @@ static void tag_set_flatten(struct tag_set *tags) { memmove(dbase, kvp, (size_t)copy_size); tags->kvm_used -= (size_t)reduce; next_kvp -= reduce; - find_deleted = true; + found_deleted = false; + } + } else { + if (CENSUS_TAG_IS_DELETED(tag.flags)) { + dbase = kvp; + found_deleted = true; } } kvp = next_kvp; } - if (!find_deleted) { + if (found_deleted) { GPR_ASSERT(dbase > tags->kvm); tags->kvm_used = (size_t)(dbase - tags->kvm); } From 6670add667d9341f065550719a209f09923fce5f Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Thu, 21 Jan 2016 17:21:53 -0800 Subject: [PATCH 21/42] keep status with tag set, eliminate tag_set_ntags --- include/grpc/census.h | 16 +++--- src/core/census/tag_set.c | 79 +++++++++++++----------------- test/core/census/tag_set_test.c | 86 +++++++++++++-------------------- 3 files changed, 75 insertions(+), 106 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 5443488fbe4..ae4bf36b1b2 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -392,16 +392,16 @@ typedef struct { creation. @return A new, valid census_tag_set. */ -census_tag_set *census_tag_set_create(const census_tag_set *base, - const census_tag *tags, int ntags, - census_tag_set_create_status *status); +census_tag_set *census_tag_set_create( + const census_tag_set *base, const census_tag *tags, int ntags, + census_tag_set_create_status const **status); /* Destroy a tag set created by census_tag_set_create(). Once this function has been called, the tag set cannot be reused. */ void census_tag_set_destroy(census_tag_set *tags); -/* Get the total number of tags in the tag set. */ -int census_tag_set_ntags(const census_tag_set *tags); +const census_tag_set_create_status *census_tag_set_get_create_status( + const census_tag_set *tags); /* Structure used for tag set iteration. API clients should not use or reference internal fields - neither their contents or presence/absence are @@ -445,11 +445,9 @@ size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, char *buffer, size_t buf_size); /* Decode tag set buffers encoded with census_tag_set_encode_*(). Returns NULL - if there is an error in parsing either buffer. The number of tags in the - decoded tag set will be returned in status, if it is non-NULL. */ + if there is an error in parsing either buffer. */ census_tag_set *census_tag_set_decode(const char *buffer, size_t size, - const char *bin_buffer, size_t bin_size, - census_tag_set_create_status *status); + const char *bin_buffer, size_t bin_size); /* Get a contexts tag set. */ census_tag_set *census_context_tag_set(census_context *context); diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 5cc777b085f..88269cf406d 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -113,6 +113,7 @@ struct raw_tag { // encoding/decoding. struct census_tag_set { struct tag_set tags[3]; + census_tag_set_create_status status; }; // Indices into the tags member of census_tag_set @@ -202,8 +203,7 @@ static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, // Add/modify/delete a tag to/in a census_tag_set. Caller must validate that // tag key etc. are valid. static void cts_modify_tag(census_tag_set *tags, const census_tag *tag, - size_t key_len, - census_tag_set_create_status *status) { + size_t key_len) { // First delete the tag if it is already present. bool deleted = cts_delete_tag(tags, tag, key_len); // Determine if we need to add it back. @@ -221,19 +221,17 @@ static void cts_modify_tag(census_tag_set *tags, const census_tag *tag, added = tag_set_add_tag(&tags->tags[LOCAL_TAGS], tag, key_len); } } - if (status) { - if (deleted) { - if (call_add) { - status->n_modified_tags++; - } else { - status->n_deleted_tags++; - } + if (deleted) { + if (call_add) { + tags->status.n_modified_tags++; } else { - if (added) { - status->n_added_tags++; - } else { - status->n_ignored_tags++; - } + tags->status.n_deleted_tags++; + } + } else { + if (added) { + tags->status.n_added_tags++; + } else { + tags->status.n_ignored_tags++; } } } @@ -280,13 +278,9 @@ static void tag_set_flatten(struct tag_set *tags) { tags->ntags_alloc = tags->ntags; } -census_tag_set *census_tag_set_create(const census_tag_set *base, - const census_tag *tags, int ntags, - census_tag_set_create_status *status) { - int n_invalid_tags = 0; - if (status) { - memset(status, 0, sizeof(*status)); - } +census_tag_set *census_tag_set_create( + const census_tag_set *base, const census_tag *tags, int ntags, + census_tag_set_create_status const **status) { census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); // If we are given a base, copy it into our new tag set. Otherwise set it // to zero/NULL everything. @@ -297,6 +291,7 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, tag_set_copy(&new_ts->tags[PROPAGATED_BINARY_TAGS], &base->tags[PROPAGATED_BINARY_TAGS]); tag_set_copy(&new_ts->tags[LOCAL_TAGS], &base->tags[LOCAL_TAGS]); + memset(&new_ts->status, 0, sizeof(new_ts->status)); } // Walk over the additional tags and, for those that aren't invalid, modify // the tag set to add/replace/delete as required. @@ -306,25 +301,30 @@ census_tag_set *census_tag_set_create(const census_tag_set *base, // ignore the tag if it is too long/short. if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN && tag->value_len <= CENSUS_MAX_TAG_KV_LEN) { - cts_modify_tag(new_ts, tag, key_len, status); + cts_modify_tag(new_ts, tag, key_len); } else { - n_invalid_tags++; + new_ts->status.n_invalid_tags++; } } // Remove any deleted tags, update status if needed, and return. tag_set_flatten(&new_ts->tags[PROPAGATED_TAGS]); tag_set_flatten(&new_ts->tags[PROPAGATED_BINARY_TAGS]); tag_set_flatten(&new_ts->tags[LOCAL_TAGS]); - if (status != NULL) { - status->n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; - status->n_propagated_binary_tags = - new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; - status->n_local_tags = new_ts->tags[LOCAL_TAGS].ntags; - status->n_invalid_tags = n_invalid_tags; + new_ts->status.n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; + new_ts->status.n_propagated_binary_tags = + new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; + new_ts->status.n_local_tags = new_ts->tags[LOCAL_TAGS].ntags; + if (status) { + *status = &new_ts->status; } return new_ts; } +const census_tag_set_create_status *census_tag_set_get_create_status( + const census_tag_set *tags) { + return &tags->status; +} + void census_tag_set_destroy(census_tag_set *tags) { gpr_free(tags->tags[PROPAGATED_TAGS].kvm); gpr_free(tags->tags[PROPAGATED_BINARY_TAGS].kvm); @@ -332,12 +332,6 @@ void census_tag_set_destroy(census_tag_set *tags) { gpr_free(tags); } -int census_tag_set_ntags(const census_tag_set *tags) { - return tags->tags[PROPAGATED_TAGS].ntags + - tags->tags[PROPAGATED_BINARY_TAGS].ntags + - tags->tags[LOCAL_TAGS].ntags; -} - // Initialize a tag set iterator. Must be called before first use of the // iterator. void census_tag_set_initialize_iterator(const census_tag_set *tags, @@ -507,11 +501,7 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, } census_tag_set *census_tag_set_decode(const char *buffer, size_t size, - const char *bin_buffer, size_t bin_size, - census_tag_set_create_status *status) { - if (status) { - memset(status, 0, sizeof(*status)); - } + const char *bin_buffer, size_t bin_size) { census_tag_set *new_ts = gpr_malloc(sizeof(census_tag_set)); memset(&new_ts->tags[LOCAL_TAGS], 0, sizeof(struct tag_set)); if (buffer == NULL) { @@ -524,11 +514,10 @@ census_tag_set *census_tag_set_decode(const char *buffer, size_t size, } else { tag_set_decode(&new_ts->tags[PROPAGATED_BINARY_TAGS], bin_buffer, bin_size); } - if (status) { - status->n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; - status->n_propagated_binary_tags = - new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; - } + memset(&new_ts->status, 0, sizeof(new_ts->status)); + new_ts->status.n_propagated_tags = new_ts->tags[PROPAGATED_TAGS].ntags; + new_ts->status.n_propagated_binary_tags = + new_ts->tags[PROPAGATED_BINARY_TAGS].ntags; // TODO(aveitch): check that BINARY flag is correct for each type. return new_ts; } diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index bc084ec04bd..752a74d4016 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -109,18 +109,21 @@ static bool validate_tag(const census_tag_set *cts, const census_tag *tag) { // Create an empty tag_set. static void empty_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, NULL, 0, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == 0); + GPR_ASSERT(cts != NULL); + const census_tag_set_create_status *status = + census_tag_set_get_create_status(cts); + census_tag_set_create_status expected = {0, 0, 0, 0, 0, 0, 0, 0}; + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_destroy(cts); } // Test create and iteration over basic tag set. static void basic_test(void) { - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); census_tag_set_create_status expected = {2, 2, 4, 0, 8, 0, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_iterator it; census_tag_set_initialize_iterator(cts, &it); census_tag tag; @@ -139,9 +142,8 @@ static void basic_test(void) { static void lookup_by_key_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); census_tag tag; - for (int i = 0; i < census_tag_set_ntags(cts); i++) { + for (int i = 0; i < BASIC_TAG_COUNT; i++) { GPR_ASSERT(census_tag_set_get_tag_by_key(cts, basic_tags[i].key, &tag) == 1); GPR_ASSERT(compare_tag(&tag, &basic_tags[i])); @@ -166,46 +168,39 @@ static void invalid_test(void) { // long keys, short value. Key lengths (including terminator) should be // <= 255 (CENSUS_MAX_TAG_KV_LEN) GPR_ASSERT(strlen(key) == 299); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 0); census_tag_set_create_status expected = {0, 0, 0, 0, 0, 0, 1, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_destroy(cts); key[CENSUS_MAX_TAG_KV_LEN] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN); cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 0); - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_destroy(cts); key[CENSUS_MAX_TAG_KV_LEN - 1] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN - 1); cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 1); census_tag_set_create_status expected2 = {0, 0, 1, 0, 1, 0, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected2, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected2, sizeof(expected2)) == 0); census_tag_set_destroy(cts); // now try with long values tag.value_len = 300; cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 0); - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_destroy(cts); tag.value_len = CENSUS_MAX_TAG_KV_LEN + 1; cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 0); - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_destroy(cts); tag.value_len = CENSUS_MAX_TAG_KV_LEN; cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 1); - GPR_ASSERT(memcmp(&status, &expected2, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected2, sizeof(expected2)) == 0); census_tag_set_destroy(cts); // 0 length key. key[0] = 0; cts = census_tag_set_create(NULL, &tag, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts) == 0); - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag_set_destroy(cts); } @@ -213,13 +208,11 @@ static void invalid_test(void) { static void copy_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0, &status); - GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); census_tag_set_create_status expected = {2, 2, 4, 0, 0, 0, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); - for (int i = 0; i < census_tag_set_ntags(cts2); i++) { + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); + for (int i = 0; i < BASIC_TAG_COUNT; i++) { census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) == 1); @@ -233,13 +226,11 @@ static void copy_test(void) { static void replace_value_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts2 = census_tag_set_create( cts, modify_tags + REPLACE_VALUE_OFFSET, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); census_tag_set_create_status expected = {2, 2, 4, 0, 0, 1, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[REPLACE_VALUE_OFFSET].key, &tag) == 1); @@ -252,13 +243,11 @@ static void replace_value_test(void) { static void replace_flags_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts2 = census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT); census_tag_set_create_status expected = {1, 2, 5, 0, 0, 1, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[REPLACE_FLAG_OFFSET].key, &tag) == 1); @@ -271,13 +260,11 @@ static void replace_flags_test(void) { static void delete_tag_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts2 = census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT - 1); census_tag_set_create_status expected = {2, 1, 4, 1, 0, 0, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[DELETE_TAG_OFFSET].key, &tag) == 0); @@ -289,13 +276,11 @@ static void delete_tag_test(void) { static void add_tag_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts2 = census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1, &status); - GPR_ASSERT(census_tag_set_ntags(cts2) == BASIC_TAG_COUNT + 1); census_tag_set_create_status expected = {2, 2, 5, 0, 1, 0, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_tag_set_get_tag_by_key( cts2, modify_tags[ADD_TAG_OFFSET].key, &tag) == 1); @@ -308,13 +293,11 @@ static void add_tag_test(void) { static void replace_add_delete_test(void) { struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); - census_tag_set_create_status status; + const census_tag_set_create_status *status; struct census_tag_set *cts2 = census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, &status); - GPR_ASSERT(census_tag_set_ntags(cts2) == 9); census_tag_set_create_status expected = {2, 1, 6, 2, 3, 4, 0, 2}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); // validate tag set contents. Use specific indices into the two arrays // holding tag values. GPR_ASSERT(validate_tag(cts2, &basic_tags[3])); @@ -342,20 +325,19 @@ static void encode_decode_test(void) { char buf2[1000]; struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_ntags(cts) == BASIC_TAG_COUNT); GPR_ASSERT(census_tag_set_encode_propagated(cts, buf1, 1) == 0); size_t b1 = census_tag_set_encode_propagated(cts, buf1, 1000); GPR_ASSERT(b1 != 0); GPR_ASSERT(census_tag_set_encode_propagated_binary(cts, buf2, 1) == 0); size_t b2 = census_tag_set_encode_propagated_binary(cts, buf2, 1000); GPR_ASSERT(b2 != 0); - census_tag_set_create_status status; - census_tag_set *cts2 = census_tag_set_decode(buf1, b1, buf2, b2, &status); + census_tag_set *cts2 = census_tag_set_decode(buf1, b1, buf2, b2); GPR_ASSERT(cts2 != NULL); - GPR_ASSERT(census_tag_set_ntags(cts2) == 4); + const census_tag_set_create_status *status = + census_tag_set_get_create_status(cts2); census_tag_set_create_status expected = {2, 2, 0, 0, 0, 0, 0, 0}; - GPR_ASSERT(memcmp(&status, &expected, sizeof(status)) == 0); - for (int i = 0; i < census_tag_set_ntags(cts); i++) { + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); + for (int i = 0; i < BASIC_TAG_COUNT; i++) { census_tag tag; if (CENSUS_TAG_IS_PROPAGATED(basic_tags[i].flags)) { GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) == From 0c1cdcd38aabf0d1218b1360665562ff1813b64d Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Thu, 21 Jan 2016 17:25:41 -0800 Subject: [PATCH 22/42] update comments --- include/grpc/census.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index ae4bf36b1b2..84a3f8faf69 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -387,9 +387,9 @@ typedef struct { binary or non-binary tags, they will be ignored, as will deletions of tags that don't exist. @param ntags number of tags in 'tags' - @param status If not NULL, the pointed to structure will be filled in with - information about the new tag set and status of the tags used in its - creation. + @param status If not NULL, will return a pointer to a + census_tag_set_create_status structure containing information about the new + tag set and status of the tags used in its creation. @return A new, valid census_tag_set. */ census_tag_set *census_tag_set_create( @@ -400,6 +400,7 @@ census_tag_set *census_tag_set_create( has been called, the tag set cannot be reused. */ void census_tag_set_destroy(census_tag_set *tags); +/* Get a pointer to the original status from the creation of this tag set. */ const census_tag_set_create_status *census_tag_set_get_create_status( const census_tag_set *tags); From 25fe36ac83f33755c3062a365cd1b80235b3617f Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 21 Jan 2016 21:33:55 -0800 Subject: [PATCH 23/42] Revert run_tests.py to master. --- tools/run_tests/run_tests.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index e86467e3f23..637aff8585b 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -115,13 +115,10 @@ class ValgrindConfig(object): self.args = args self.allow_hashing = False - def job_spec(self, cmdline, hash_targets, cpu_cost=1.0, timeout_seconds=None, - shortname=None, environ=None): - if shortname is None: - shortname = 'valgrind %s' % cmdline[0] + def job_spec(self, cmdline, hash_targets, cpu_cost=1.0): return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] + self.args + cmdline, - shortname=shortname, + shortname='valgrind %s' % cmdline[0], hash_targets=None, cpu_cost=cpu_cost, flake_retries=5 if args.allow_flakes else 0, From d409e3bf41d0102c7ab4370a5470c4259aad88a8 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 22 Jan 2016 09:34:23 -0800 Subject: [PATCH 24/42] update comments --- src/core/census/tag_set.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 88269cf406d..f573e909b3a 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -72,19 +72,22 @@ struct tag_set { int ntags; // number of tags. int ntags_alloc; // ntags + number of deleted tags (total number of tags - // in all of kvm). This will always be == ntags, except - // during the process of building a new tag set. + // in all of kvm). This will always be == ntags, except during the process + // of building a new tag set. size_t kvm_size; // number of bytes allocated for key/value storage. size_t kvm_used; // number of bytes of used key/value memory char *kvm; // key/value memory. Consists of repeated entries of: - // Offset Size Description - // 0 1 Key length, including trailing 0. (K) - // 1 1 Value length. (V) - // 2 1 Flags - // 3 K Key bytes - // 3 + K V Value bytes - // - // We refer to the first 3 entries as the 'tag header'. + // Offset Size Description + // 0 1 Key length, including trailing 0. (K) + // 1 1 Value length. (V) + // 2 1 Flags + // 3 K Key bytes + // 3 + K V Value bytes + // + // We refer to the first 3 entries as the 'tag header'. If extra values are + // introduced in the header, you will need to modify the TAG_HEADER_SIZE + // constant, the raw_tag structure (and everything that uses it) and the + // encode/decode functions appropriately. }; // Number of bytes in tag header. @@ -170,7 +173,9 @@ static bool cts_delete_tag(census_tag_set *tags, const census_tag *tag, } // Add a tag to a tag_set. Return true on sucess, false if the tag could -// not be added because of constraints on tag set size. +// not be added because of constraints on tag set size. This function should +// not be called if the tag may already exist (in a non-deleted state) in +// the tag_set, as that would result in two tags with the same key. static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, size_t key_len) { if (tags->ntags == CENSUS_MAX_PROPAGATED_TAGS) { From c45d088ae7d8f67f9422abc19562a81307bf78a2 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 22 Jan 2016 11:43:30 -0800 Subject: [PATCH 25/42] Single encode function --- include/grpc/census.h | 29 ++++++++++++++++++----------- src/core/census/tag_set.c | 23 +++++++++++++++-------- test/core/census/tag_set_test.c | 20 +++++++++++--------- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 84a3f8faf69..6e4fa5a8258 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -433,17 +433,24 @@ int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, for use by RPC systems only, for purposes of transmitting/receiving tag sets. */ -/* Encode to-be-propagated non-binary tags from a tag set into a memory - buffer. The total number of bytes used in the buffer is returned. If the - buffer is too small to contain the encoded tag set, then 0 is returned. */ -size_t census_tag_set_encode_propagated(const census_tag_set *tags, - char *buffer, size_t buf_size); - -/* Encode to-be-propagated binary tags from a tag set into a memory - buffer. The total number of bytes used in the buffer is returned. If the - buffer is too small to contain the encoded tag set, then 0 is returned. */ -size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, - char *buffer, size_t buf_size); +/* Encode a tag set into a buffer. The propagated tags are encoded into the + buffer in two regions: one for printable tags, and one for binary tags. + @param tags tag set to be encoded + @param buffer pointer to buffer. This address will be used to encode the + printable tags. + @param buf_size On input, will be a pointer to total buffer size. On output, + will be set to total number of bytes consumed by printable + tags. + @param bin_buf_size on output, will be set to the number of bytes used to + encode the binary tags. + @return A pointer to the binary tag's encoded, or NULL if the buffer was + insufficiently large to hold the encoded tags. Thus, if successful, + printable tags are encoded into + [buffer, buffer + *buf_size) and binary tags into + [returned-ptr, returned-ptr + *bin_buf_size) (and the return value + should be buffer + *buf_size) */ +char *census_tag_set_encode(const census_tag_set *tags, char *buffer, + size_t *buf_size, size_t *bin_buf_size); /* Decode tag set buffers encoded with census_tag_set_encode_*(). Returns NULL if there is an error in parsing either buffer. */ diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index f573e909b3a..b985eca52b2 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -453,14 +453,21 @@ static size_t tag_set_encode(const struct tag_set *tags, char *buffer, return ENCODED_HEADER_SIZE + tags->kvm_used; } -size_t census_tag_set_encode_propagated(const census_tag_set *tags, - char *buffer, size_t buf_size) { - return tag_set_encode(&tags->tags[PROPAGATED_TAGS], buffer, buf_size); -} - -size_t census_tag_set_encode_propagated_binary(const census_tag_set *tags, - char *buffer, size_t buf_size) { - return tag_set_encode(&tags->tags[PROPAGATED_BINARY_TAGS], buffer, buf_size); +char *census_tag_set_encode(const census_tag_set *tags, char *buffer, + size_t *buf_size, size_t *bin_buf_size) { + size_t p_buf_size = + tag_set_encode(&tags->tags[PROPAGATED_TAGS], buffer, *buf_size); + if (p_buf_size == 0) { + return NULL; + } + char *b_buffer = buffer + p_buf_size; + *bin_buf_size = tag_set_encode(&tags->tags[PROPAGATED_BINARY_TAGS], b_buffer, + *buf_size - p_buf_size); + if (*bin_buf_size == 0) { + return NULL; + } + *buf_size = p_buf_size; + return b_buffer; } // Decode a tag set. diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index 752a74d4016..4414ad9bc18 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -321,17 +321,19 @@ static void replace_add_delete_test(void) { // test encode/decode. static void encode_decode_test(void) { - char buf1[1000]; - char buf2[1000]; + const size_t BUF_SIZE = 200; + char buffer[BUF_SIZE]; struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - GPR_ASSERT(census_tag_set_encode_propagated(cts, buf1, 1) == 0); - size_t b1 = census_tag_set_encode_propagated(cts, buf1, 1000); - GPR_ASSERT(b1 != 0); - GPR_ASSERT(census_tag_set_encode_propagated_binary(cts, buf2, 1) == 0); - size_t b2 = census_tag_set_encode_propagated_binary(cts, buf2, 1000); - GPR_ASSERT(b2 != 0); - census_tag_set *cts2 = census_tag_set_decode(buf1, b1, buf2, b2); + size_t bsize = 2; // buffer size too small + size_t bin_bsize = 0; + GPR_ASSERT(census_tag_set_encode(cts, buffer, &bsize, &bin_bsize) == NULL); + bsize = BUF_SIZE; + char *b_buffer = census_tag_set_encode(cts, buffer, &bsize, &bin_bsize); + GPR_ASSERT(b_buffer != NULL && bsize > 0 && bin_bsize > 0 && + bsize + bin_bsize <= BUF_SIZE && b_buffer == buffer + bsize); + census_tag_set *cts2 = + census_tag_set_decode(buffer, bsize, b_buffer, bin_bsize); GPR_ASSERT(cts2 != NULL); const census_tag_set_create_status *status = census_tag_set_get_create_status(cts2); From ff14b44154b4667e666531763898e503f9f00bfd Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 22 Jan 2016 13:01:49 -0800 Subject: [PATCH 26/42] Don't use buf_size as input and output in encode --- include/grpc/census.h | 17 +++++++++-------- src/core/census/tag_set.c | 14 +++++++------- test/core/census/tag_set_test.c | 19 +++++++++++-------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 6e4fa5a8258..ab0e0e48026 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -438,19 +438,20 @@ int census_tag_set_get_tag_by_key(const census_tag_set *tags, const char *key, @param tags tag set to be encoded @param buffer pointer to buffer. This address will be used to encode the printable tags. - @param buf_size On input, will be a pointer to total buffer size. On output, - will be set to total number of bytes consumed by printable - tags. - @param bin_buf_size on output, will be set to the number of bytes used to - encode the binary tags. + @param buf_size number of available bytes in buffer. + @param print_buf_size Will be set to the number of bytes consumed by + printable tags. + @param bin_buf_size Will be set to the number of bytes used to encode the + binary tags. @return A pointer to the binary tag's encoded, or NULL if the buffer was insufficiently large to hold the encoded tags. Thus, if successful, printable tags are encoded into - [buffer, buffer + *buf_size) and binary tags into + [buffer, buffer + *print_buf_size) and binary tags into [returned-ptr, returned-ptr + *bin_buf_size) (and the return value - should be buffer + *buf_size) */ + should be buffer + *print_buf_size) */ char *census_tag_set_encode(const census_tag_set *tags, char *buffer, - size_t *buf_size, size_t *bin_buf_size); + size_t buf_size, size_t *print_buf_size, + size_t *bin_buf_size); /* Decode tag set buffers encoded with census_tag_set_encode_*(). Returns NULL if there is an error in parsing either buffer. */ diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index b985eca52b2..8908a2d5f3c 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -454,19 +454,19 @@ static size_t tag_set_encode(const struct tag_set *tags, char *buffer, } char *census_tag_set_encode(const census_tag_set *tags, char *buffer, - size_t *buf_size, size_t *bin_buf_size) { - size_t p_buf_size = - tag_set_encode(&tags->tags[PROPAGATED_TAGS], buffer, *buf_size); - if (p_buf_size == 0) { + size_t buf_size, size_t *print_buf_size, + size_t *bin_buf_size) { + *print_buf_size = + tag_set_encode(&tags->tags[PROPAGATED_TAGS], buffer, buf_size); + if (*print_buf_size == 0) { return NULL; } - char *b_buffer = buffer + p_buf_size; + char *b_buffer = buffer + *print_buf_size; *bin_buf_size = tag_set_encode(&tags->tags[PROPAGATED_BINARY_TAGS], b_buffer, - *buf_size - p_buf_size); + buf_size - *print_buf_size); if (*bin_buf_size == 0) { return NULL; } - *buf_size = p_buf_size; return b_buffer; } diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index 4414ad9bc18..8e09e6c1c68 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -325,15 +325,18 @@ static void encode_decode_test(void) { char buffer[BUF_SIZE]; struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - size_t bsize = 2; // buffer size too small - size_t bin_bsize = 0; - GPR_ASSERT(census_tag_set_encode(cts, buffer, &bsize, &bin_bsize) == NULL); - bsize = BUF_SIZE; - char *b_buffer = census_tag_set_encode(cts, buffer, &bsize, &bin_bsize); - GPR_ASSERT(b_buffer != NULL && bsize > 0 && bin_bsize > 0 && - bsize + bin_bsize <= BUF_SIZE && b_buffer == buffer + bsize); + size_t print_bsize; + size_t bin_bsize; + // Test with too small a buffer + GPR_ASSERT(census_tag_set_encode(cts, buffer, 2, &print_bsize, &bin_bsize) == + NULL); + char *b_buffer = + census_tag_set_encode(cts, buffer, BUF_SIZE, &print_bsize, &bin_bsize); + GPR_ASSERT(b_buffer != NULL && print_bsize > 0 && bin_bsize > 0 && + print_bsize + bin_bsize <= BUF_SIZE && + b_buffer == buffer + print_bsize); census_tag_set *cts2 = - census_tag_set_decode(buffer, bsize, b_buffer, bin_bsize); + census_tag_set_decode(buffer, print_bsize, b_buffer, bin_bsize); GPR_ASSERT(cts2 != NULL); const census_tag_set_create_status *status = census_tag_set_get_create_status(cts2); From 72f9a67cea8ea075b4f7b553845c6292e5aacf8b Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 22 Jan 2016 14:27:58 -0800 Subject: [PATCH 27/42] Fix path where we check for ruby coverage report --- tools/run_tests/post_tests_ruby.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/post_tests_ruby.sh b/tools/run_tests/post_tests_ruby.sh index 66a9fbc5348..0877e44805a 100755 --- a/tools/run_tests/post_tests_ruby.sh +++ b/tools/run_tests/post_tests_ruby.sh @@ -43,4 +43,4 @@ genhtml $tmp2 --output-directory $out rm $tmp2 rm $tmp1 -cp -rv $root/src/ruby/coverage $root/reports/ruby +cp -rv $root/coverage $root/reports/ruby From 73128da6519f99669466055d6d29533343437031 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Jan 2016 14:48:10 -0800 Subject: [PATCH 28/42] fix copyrights --- examples/objective-c/auth_sample/MakeRPCViewController.m | 2 +- src/objective-c/examples/Sample/Sample/ViewController.m | 2 +- src/objective-c/tests/InteropTestsRemote.m | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/objective-c/auth_sample/MakeRPCViewController.m b/examples/objective-c/auth_sample/MakeRPCViewController.m index 6013186b994..108eda45ba5 100644 --- a/examples/objective-c/auth_sample/MakeRPCViewController.m +++ b/examples/objective-c/auth_sample/MakeRPCViewController.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/objective-c/examples/Sample/Sample/ViewController.m b/src/objective-c/examples/Sample/Sample/ViewController.m index 433a8a2ba3b..a2bb3ee3298 100644 --- a/src/objective-c/examples/Sample/Sample/ViewController.m +++ b/src/objective-c/examples/Sample/Sample/ViewController.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/objective-c/tests/InteropTestsRemote.m b/src/objective-c/tests/InteropTestsRemote.m index 758cc9346ad..00eadc25bc3 100644 --- a/src/objective-c/tests/InteropTestsRemote.m +++ b/src/objective-c/tests/InteropTestsRemote.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 38b06fbc3519352f97ad8ea6fb817e80b93e7451 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Jan 2016 14:58:24 -0800 Subject: [PATCH 29/42] make sure we embed openssl and zlib in artifacts --- tools/run_tests/build_artifacts.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/run_tests/build_artifacts.py b/tools/run_tests/build_artifacts.py index 0fd5bc63f37..ff9dd4735ac 100755 --- a/tools/run_tests/build_artifacts.py +++ b/tools/run_tests/build_artifacts.py @@ -129,18 +129,19 @@ class CSharpExtArtifact: '/p:PlatformToolset=v120', '/p:Platform=%s' % msbuild_platform], shell=True) - if self.platform == 'linux': - environ = {'CONFIG': 'opt'} - return create_docker_jobspec(self.name, - 'tools/jenkins/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_csharp.sh') else: - environ = {'CONFIG': 'opt'} - if self.platform == 'macos': + environ = {'CONFIG': 'opt', + 'EMBED_OPENSSL': 'true', + 'EMBED_ZLIB': 'true'} + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/jenkins/grpc_artifact_linux_%s' % self.arch, + 'tools/run_tests/build_artifact_csharp.sh') + else: environ.update(macos_arch_env(self.arch)) - return create_jobspec(self.name, - ['tools/run_tests/build_artifact_csharp.sh'], - environ=environ) + return create_jobspec(self.name, + ['tools/run_tests/build_artifact_csharp.sh'], + environ=environ) def __str__(self): return self.name From f968a87e3f54711b92d3906877b9f945d9f9a120 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 22 Jan 2016 16:07:42 -0800 Subject: [PATCH 30/42] Fix build --- grpc.gemspec | 1 + package.json | 1 + tools/run_tests/tests.json | 1 + vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj | 2 ++ 4 files changed, 5 insertions(+) diff --git a/grpc.gemspec b/grpc.gemspec index 4f3a3f2dd91..47b66ae535d 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -412,5 +412,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/census/context.c ) s.files += %w( src/core/census/initialize.c ) s.files += %w( src/core/census/operation.c ) + s.files += %w( src/core/census/tag_set.c ) s.files += %w( src/core/census/tracing.c ) end diff --git a/package.json b/package.json index e0bbb7d80ff..0b87f6a59d8 100644 --- a/package.json +++ b/package.json @@ -363,6 +363,7 @@ "src/core/census/context.c", "src/core/census/initialize.c", "src/core/census/operation.c", + "src/core/census/tag_set.c", "src/core/census/tracing.c", "include/grpc/support/alloc.h", "include/grpc/support/atm.h", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 5124becfbe7..8c76b3f134f 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1407,6 +1407,7 @@ "posix", "windows" ], + "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, "language": "c", diff --git a/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj b/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj index 38949026726..9a468af3adc 100644 --- a/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj +++ b/vsprojects/vcxproj/test/tag_set_test/tag_set_test.vcxproj @@ -63,12 +63,14 @@ tag_set_test static Debug + static Debug tag_set_test static Release + static Release From 58a1dc2bb31f4bf3581131fba80497c40ddc947d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 21 Jan 2016 14:23:55 -0800 Subject: [PATCH 31/42] Add compiler error diagnostics to Python setup --- PYTHON-MANIFEST.in | 1 + setup.py | 2 +- src/python/grpcio/commands.py | 10 +++- src/python/grpcio/support.py | 91 +++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 src/python/grpcio/support.py diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 02bd9b52294..6e54e1b3c64 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -5,6 +5,7 @@ graft include/grpc graft third_party/boringssl include src/python/grpcio/commands.py include src/python/grpcio/grpc_core_dependencies.py +include src/python/grpcio/support.py include src/python/grpcio/README.rst include requirements.txt include etc/roots.pem diff --git a/setup.py b/setup.py index 63b56f35327..113e49ecfe5 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ BORINGSSL_INCLUDE = ('./third_party/boringssl/include',) # Ensure we're in the proper directory whether or not we're being used by pip. os.chdir(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, PYTHON_STEM) +sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. import commands diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index bd12c5579cb..ff35c458617 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -37,9 +37,9 @@ import subprocess import sys import setuptools +from setuptools.command import build_ext from setuptools.command import build_py from setuptools.command import test -from setuptools.command import build_ext PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) @@ -186,7 +186,13 @@ class BuildExt(build_ext.build_ext): if compiler in BuildExt.LINK_OPTIONS: for extension in self.extensions: extension.extra_link_args += list(BuildExt.LINK_OPTIONS[compiler]) - build_ext.build_ext.build_extensions(self) + try: + build_ext.build_ext.build_extensions(self) + except KeyboardInterrupt: + raise + except Exception as error: + support.diagnose_build_ext_error(self, error) + raise CommandError("Failed `build_ext` step.") class Gather(setuptools.Command): diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py new file mode 100644 index 00000000000..bbc509653d1 --- /dev/null +++ b/src/python/grpcio/support.py @@ -0,0 +1,91 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import os.path +import shutil +import sys +import tempfile + +from distutils import errors + +import commands + + +C_PYTHON_DEV = """ +#include +int main(int argc, char **argv) { return 0; } +""" +C_PYTHON_DEV_ERROR_MESSAGE = """ +Could not find . This could mean the following: + * You're on Ubuntu and haven't `apt-get install`ed `python-dev`. + * You're on Mac OS X and the usual Python framework was somehow corrupted + (check your environment variables or try re-installing?) + * You're on Windows and your Python installation was somehow corrupted + (check your environment variables or try re-installing?) + * Note: Windows users should look into installing `vcpython27`. +""" + +C_CHECKS = { + C_PYTHON_DEV: C_PYTHON_DEV_ERROR_MESSAGE, +} + +def _compile(compiler, source_string): + tempdir = tempfile.mkdtemp() + cpath = os.path.join(tempdir, 'a.c') + with open(cpath, 'w') as cfile: + cfile.write(source_string) + try: + compiler.compile([cpath]) + except errors.CompileError as error: + return error + finally: + shutil.rmtree(tempdir) + +def _expect_compile(compiler, source_string, error_message): + if _compile(compiler, source_string) is not None: + sys.stderr.write(error_message) + raise commands.CommandError( + "Diagnostics found a compilation environment issue:\n{}" + .format(error_message)) + +def diagnose_build_ext_error(build_ext, error): + { + errors.CompileError: diagnose_compile_error + }[type(error)](build_ext, error) + +def diagnose_compile_error(build_ext, error): + """Attempt to run a few test files through the compiler to see if we can + diagnose the reason for the compile failure.""" + for c_check, message in C_CHECKS.items(): + _expect_compile(build_ext.compiler, c_check, message) + raise commands.CommandError( + "\n\nWe could not diagnose your build failure. Please file an issue at " + "http://www.github.com/grpc/grpc with `[Python install]` in the title.") From 0cc27920f8a54aa1b658be43b1b1a16ff9de21f0 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 22 Jan 2016 16:32:41 -0800 Subject: [PATCH 32/42] Integrate zlib into Python distribution --- PYTHON-MANIFEST.in | 1 + build.yaml | 1 + setup.py | 5 +++-- src/python/grpcio/grpc_core_dependencies.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 02bd9b52294..52ef1aba5b3 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -3,6 +3,7 @@ graft src/python/grpcio/tests graft src/core graft include/grpc graft third_party/boringssl +graft third_party/zlib include src/python/grpcio/commands.py include src/python/grpcio/grpc_core_dependencies.py include src/python/grpcio/README.rst diff --git a/build.yaml b/build.yaml index c05dde812e0..500ec855ae4 100644 --- a/build.yaml +++ b/build.yaml @@ -2636,3 +2636,4 @@ python_dependencies: - grpc - gpr - boringssl + - z diff --git a/setup.py b/setup.py index 63b56f35327..c54ac221297 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in' PYTHON_STEM = './src/python/grpcio' CORE_INCLUDE = ('./include', '.',) BORINGSSL_INCLUDE = ('./third_party/boringssl/include',) +ZLIB_INCLUDE = ('./third_party/zlib',) # Ensure we're in the proper directory whether or not we're being used by pip. os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -75,9 +76,9 @@ CYTHON_EXTENSION_PACKAGE_NAMES = () CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',) EXTENSION_INCLUDE_DIRECTORIES = ( - (PYTHON_STEM,) + CORE_INCLUDE + BORINGSSL_INCLUDE) + (PYTHON_STEM,) + CORE_INCLUDE + BORINGSSL_INCLUDE + ZLIB_INCLUDE) -EXTENSION_LIBRARIES = ('z', 'm',) +EXTENSION_LIBRARIES = ('m',) if not "darwin" in sys.platform: EXTENSION_LIBRARIES += ('rt',) diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 66a55ef3e58..98ab1ff7f0e 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -517,4 +517,19 @@ CORE_SOURCE_FILES = [ 'third_party/boringssl/ssl/t1_enc.c', 'third_party/boringssl/ssl/t1_lib.c', 'third_party/boringssl/ssl/tls_record.c', + 'third_party/zlib/adler32.c', + 'third_party/zlib/compress.c', + 'third_party/zlib/crc32.c', + 'third_party/zlib/deflate.c', + 'third_party/zlib/gzclose.c', + 'third_party/zlib/gzlib.c', + 'third_party/zlib/gzread.c', + 'third_party/zlib/gzwrite.c', + 'third_party/zlib/infback.c', + 'third_party/zlib/inffast.c', + 'third_party/zlib/inflate.c', + 'third_party/zlib/inftrees.c', + 'third_party/zlib/trees.c', + 'third_party/zlib/uncompr.c', + 'third_party/zlib/zutil.c', ] From 5ff8de34529915e3f0256305eccfc5d69353413d Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 22 Jan 2016 16:55:10 -0800 Subject: [PATCH 33/42] init deadline_ --- src/cpp/server/server_context.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 8193e70660d..5be2271a4e7 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -122,6 +122,7 @@ ServerContext::ServerContext() : completion_op_(nullptr), has_notify_when_done_tag_(false), async_notify_when_done_tag_(nullptr), + deadline_(gpr_inf_future(GPR_CLOCK_REALTIME)), call_(nullptr), cq_(nullptr), sent_initial_metadata_(false) {} From 03a38e1dab254d125d00cc3507202f559249c40d Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 22 Jan 2016 17:56:02 -0800 Subject: [PATCH 34/42] fix uninitialized variable usage --- src/core/census/tag_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/census/tag_set.c b/src/core/census/tag_set.c index 8908a2d5f3c..9b65a1dfa12 100644 --- a/src/core/census/tag_set.c +++ b/src/core/census/tag_set.c @@ -253,7 +253,7 @@ static void tag_set_flatten(struct tag_set *tags) { if (tags->ntags == tags->ntags_alloc) return; bool found_deleted = false; // found a deleted tag. char *kvp = tags->kvm; - char *dbase; // record location of deleted tag + char *dbase = NULL; // record location of deleted tag for (int i = 0; i < tags->ntags_alloc; i++) { struct raw_tag tag; char *next_kvp = decode_tag(&tag, kvp, 0); From 1cebbcade4656dc3efcaf62814ee8e715f5cb4bf Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Jan 2016 19:49:44 -0800 Subject: [PATCH 35/42] fix formatting --- include/grpc/census.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index ab0e0e48026..f4130c7e6a1 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -506,8 +506,8 @@ extern census_aggregation_ops census_agg_window; construction via census_define_view(). */ typedef struct { const census_aggregation_ops *ops; - const void - *create_arg; /* Argument to be used for aggregation initialization. */ + const void * + create_arg; /* Argument to be used for aggregation initialization. */ } census_aggregation; /** A census view type. Opaque. */ From 45be7ff6cfd48dd79ad19440eccdb4a053c6a6bb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Jan 2016 20:12:55 -0800 Subject: [PATCH 36/42] fix compilation on windows --- test/core/census/tag_set_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c index 8e09e6c1c68..1056e98d55c 100644 --- a/test/core/census/tag_set_test.c +++ b/test/core/census/tag_set_test.c @@ -319,9 +319,10 @@ static void replace_add_delete_test(void) { census_tag_set_destroy(cts2); } +#define BUF_SIZE 200 + // test encode/decode. static void encode_decode_test(void) { - const size_t BUF_SIZE = 200; char buffer[BUF_SIZE]; struct census_tag_set *cts = census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); From b92cb5959e89e09c62f256c25de67df88499e53c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Jan 2016 20:47:42 -0800 Subject: [PATCH 37/42] Run nuget restore on mac as well --- tools/run_tests/pre_build_csharp.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/run_tests/pre_build_csharp.sh b/tools/run_tests/pre_build_csharp.sh index 42ff60bea22..06c8a83071b 100755 --- a/tools/run_tests/pre_build_csharp.sh +++ b/tools/run_tests/pre_build_csharp.sh @@ -35,6 +35,11 @@ cd $(dirname $0)/../../src/csharp root=`pwd` +if [ -x "$(command -v nuget)" ] +then + nuget restore Grpc.sln +fi + if [ -n "$NUGET" ] then $NUGET restore Grpc.sln From c4d9f6de6644fbd773de03d133843c9c05c2503f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 23 Jan 2016 07:50:25 -0800 Subject: [PATCH 38/42] fix copyrights in server_context --- src/cpp/server/server_context.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 5be2271a4e7..b3a74c7fce6 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 3175f7c7335deb8b0801d8e5b628d6f6da48fefd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 23 Jan 2016 07:54:44 -0800 Subject: [PATCH 39/42] fix copyright --- tools/run_tests/pre_build_csharp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/pre_build_csharp.sh b/tools/run_tests/pre_build_csharp.sh index 06c8a83071b..4341c0256fd 100755 --- a/tools/run_tests/pre_build_csharp.sh +++ b/tools/run_tests/pre_build_csharp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From e399803234a4017b3ee11017b8e433cd38846c0f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 23 Jan 2016 11:50:16 -0800 Subject: [PATCH 40/42] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0553ecc861..f894def4706 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [gRPC - An RPC library and framework](http://github.com/grpc/grpc) =================================== -Copyright 2015 Google Inc. +Copyright 2015-2016 Google Inc. #Documentation From 7928b4175f0dcde511ba4c4ffc15a48f3867b633 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Jan 2016 16:38:26 -0800 Subject: [PATCH 41/42] update C# dependencies --- src/csharp/Grpc.Auth/Grpc.Auth.csproj | 21 ++++++++------- src/csharp/Grpc.Auth/packages.config | 4 +-- .../Grpc.Core.Tests/Grpc.Core.Tests.csproj | 5 ++-- src/csharp/Grpc.Core.Tests/packages.config | 2 +- src/csharp/Grpc.Core/Grpc.Core.csproj | 5 ++-- src/csharp/Grpc.Core/Grpc.Core.nuspec | 2 +- src/csharp/Grpc.Core/packages.config | 2 +- .../Grpc.Examples.Tests.csproj | 2 +- .../Grpc.Examples.Tests/packages.config | 2 +- src/csharp/Grpc.Examples/Grpc.Examples.csproj | 5 ++-- src/csharp/Grpc.Examples/packages.config | 2 +- .../Grpc.HealthCheck/Grpc.HealthCheck.csproj | 5 ++-- src/csharp/Grpc.HealthCheck/packages.config | 2 +- .../Grpc.IntegrationTesting.Client.csproj | 12 ++++----- .../packages.config | 4 +-- .../Grpc.IntegrationTesting.Server.csproj | 12 ++++----- .../packages.config | 4 +-- .../Grpc.IntegrationTesting.csproj | 26 +++++++++++-------- .../Grpc.IntegrationTesting/packages.config | 6 ++--- 19 files changed, 67 insertions(+), 56 deletions(-) diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index 7e106544a7d..8dd12b50ef5 100644 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -39,6 +39,18 @@ ..\keys\Grpc.snk + + False + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.dll + + + False + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.PlatformServices.dll + + + False + ..\packages\Google.Apis.Core.1.10.0\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll + @@ -46,15 +58,6 @@ ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.dll - - - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.PlatformServices.dll - - - ..\packages\Google.Apis.Core.1.9.3\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll diff --git a/src/csharp/Grpc.Auth/packages.config b/src/csharp/Grpc.Auth/packages.config index 7a02c95db91..5fe8ca616c7 100644 --- a/src/csharp/Grpc.Auth/packages.config +++ b/src/csharp/Grpc.Auth/packages.config @@ -1,8 +1,8 @@  - - + + diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 83078281935..7e73c4f1817 100644 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -55,8 +55,9 @@ False - - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + + False + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll diff --git a/src/csharp/Grpc.Core.Tests/packages.config b/src/csharp/Grpc.Core.Tests/packages.config index 62077f41bee..610831dfe1e 100644 --- a/src/csharp/Grpc.Core.Tests/packages.config +++ b/src/csharp/Grpc.Core.Tests/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 38398a2afd6..307c0e87837 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -39,8 +39,9 @@ - - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + + False + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll diff --git a/src/csharp/Grpc.Core/Grpc.Core.nuspec b/src/csharp/Grpc.Core/Grpc.Core.nuspec index 67a04afc22d..7582cdf965c 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.nuspec +++ b/src/csharp/Grpc.Core/Grpc.Core.nuspec @@ -15,7 +15,7 @@ Copyright 2015, Google Inc. gRPC RPC Protocol HTTP/2 - + diff --git a/src/csharp/Grpc.Core/packages.config b/src/csharp/Grpc.Core/packages.config index 86a48c512e6..80daf048d0b 100644 --- a/src/csharp/Grpc.Core/packages.config +++ b/src/csharp/Grpc.Core/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index 54c68c1f925..9aeab059f95 100644 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -47,7 +47,7 @@ False - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll diff --git a/src/csharp/Grpc.Examples.Tests/packages.config b/src/csharp/Grpc.Examples.Tests/packages.config index 10bf1e82d0c..ed30d2d428e 100644 --- a/src/csharp/Grpc.Examples.Tests/packages.config +++ b/src/csharp/Grpc.Examples.Tests/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj index c9b01950e50..15b04c8d8b7 100644 --- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj +++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj @@ -43,8 +43,9 @@ - - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + + False + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll diff --git a/src/csharp/Grpc.Examples/packages.config b/src/csharp/Grpc.Examples/packages.config index f5131b85091..dfb8304ad10 100644 --- a/src/csharp/Grpc.Examples/packages.config +++ b/src/csharp/Grpc.Examples/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj index 7ffab4b692b..498528aa18c 100644 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj @@ -44,8 +44,9 @@ - - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + + False + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll diff --git a/src/csharp/Grpc.HealthCheck/packages.config b/src/csharp/Grpc.HealthCheck/packages.config index 04eb42d1aa0..358a978ba91 100644 --- a/src/csharp/Grpc.HealthCheck/packages.config +++ b/src/csharp/Grpc.HealthCheck/packages.config @@ -1,5 +1,5 @@  - + \ No newline at end of file diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index 74d73e88d16..f37c1464c3d 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -43,17 +43,17 @@ False ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - + False - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.dll + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.dll - + False - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.PlatformServices.dll + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.PlatformServices.dll - + False - ..\packages\Google.Apis.Core.1.9.3\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll + ..\packages\Google.Apis.Core.1.10.0\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll False diff --git a/src/csharp/Grpc.IntegrationTesting.Client/packages.config b/src/csharp/Grpc.IntegrationTesting.Client/packages.config index 7a02c95db91..5fe8ca616c7 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/packages.config +++ b/src/csharp/Grpc.IntegrationTesting.Client/packages.config @@ -1,8 +1,8 @@  - - + + diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index f877b7fba26..f27b96a53f9 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -43,17 +43,17 @@ False ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - + False - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.dll + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.dll - + False - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.PlatformServices.dll + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.PlatformServices.dll - + False - ..\packages\Google.Apis.Core.1.9.3\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll + ..\packages\Google.Apis.Core.1.10.0\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll False diff --git a/src/csharp/Grpc.IntegrationTesting.Server/packages.config b/src/csharp/Grpc.IntegrationTesting.Server/packages.config index 7a02c95db91..5fe8ca616c7 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/packages.config +++ b/src/csharp/Grpc.IntegrationTesting.Server/packages.config @@ -1,8 +1,8 @@  - - + + diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index 30423313c01..372991374ee 100644 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -41,6 +41,18 @@ ..\packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll + + False + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.dll + + + False + ..\packages\Google.Apis.Auth.1.10.0\lib\net40\Google.Apis.Auth.PlatformServices.dll + + + False + ..\packages\Google.Apis.Core.1.10.0\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll + False ..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll @@ -52,8 +64,9 @@ ..\packages\NUnit.2.6.4\lib\nunit.framework.dll - - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + + False + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll @@ -61,15 +74,6 @@ ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.dll - - - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.PlatformServices.dll - - - ..\packages\Google.Apis.Core.1.9.3\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll diff --git a/src/csharp/Grpc.IntegrationTesting/packages.config b/src/csharp/Grpc.IntegrationTesting/packages.config index 68490dceec1..462dc9d6045 100644 --- a/src/csharp/Grpc.IntegrationTesting/packages.config +++ b/src/csharp/Grpc.IntegrationTesting/packages.config @@ -2,10 +2,10 @@ - - + + - + From 087543f670b2a725c6fb2236a8d7f8ca0fcc30ba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 25 Jan 2016 11:38:25 -0800 Subject: [PATCH 42/42] Fix copyright --- tools/run_tests/post_tests_ruby.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/post_tests_ruby.sh b/tools/run_tests/post_tests_ruby.sh index 0877e44805a..1a02e566c76 100755 --- a/tools/run_tests/post_tests_ruby.sh +++ b/tools/run_tests/post_tests_ruby.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without