From b35605d1da2923cb82624286feabf132cedd83ec Mon Sep 17 00:00:00 2001 From: Nicolas Noble Date: Thu, 31 Mar 2022 09:55:43 -0700 Subject: [PATCH] Adding http/1.1 support in httpcli. (#29238) * Basic HTTP/1.1 header. * Fixing format_request_test. * Adding chunked tests. * Chunked parser for HTTP/1.1 response. --- src/core/lib/http/format_request.cc | 3 +- src/core/lib/http/parser.cc | 89 ++++++++++++++++++++++++--- src/core/lib/http/parser.h | 15 ++++- test/core/http/format_request_test.cc | 8 +-- test/core/http/parser_test.cc | 28 +++++++++ 5 files changed, 127 insertions(+), 16 deletions(-) diff --git a/src/core/lib/http/format_request.cc b/src/core/lib/http/format_request.cc index 49015a971f3..685d854a61f 100644 --- a/src/core/lib/http/format_request.cc +++ b/src/core/lib/http/format_request.cc @@ -40,8 +40,7 @@ static void fill_common_header(const grpc_http_request* request, bool connection_close, std::vector* buf) { buf->push_back(path); - buf->push_back(" HTTP/1.0\r\n"); - /* just in case some crazy server really expects HTTP/1.1 */ + buf->push_back(" HTTP/1.1\r\n"); buf->push_back("Host: "); buf->push_back(host); buf->push_back("\r\n"); diff --git a/src/core/lib/http/parser.cc b/src/core/lib/http/parser.cc index d88ebdb4015..daf9a8a3191 100644 --- a/src/core/lib/http/parser.cc +++ b/src/core/lib/http/parser.cc @@ -177,6 +177,7 @@ static grpc_error_handle add_header(grpc_http_parser* parser) { uint8_t* cur = beg; uint8_t* end = beg + parser->cur_line_length; size_t* hdr_count = nullptr; + size_t size = 0; grpc_http_header** hdrs = nullptr; grpc_http_header hdr = {nullptr, nullptr}; grpc_error_handle error = GRPC_ERROR_NONE; @@ -205,13 +206,20 @@ static grpc_error_handle add_header(grpc_http_parser* parser) { cur++; } GPR_ASSERT((size_t)(end - cur) >= parser->cur_line_end_length); - hdr.value = buf2str( - cur, static_cast(end - cur) - parser->cur_line_end_length); + size = static_cast(end - cur) - parser->cur_line_end_length; + if ((size != 0) && (cur[size - 1] == '\r')) { + size--; + } + hdr.value = buf2str(cur, size); switch (parser->type) { case GRPC_HTTP_RESPONSE: hdr_count = &parser->http.response->hdr_count; hdrs = &parser->http.response->hdrs; + if ((strcmp(hdr.key, "Transfer-Encoding") == 0) && + (strcmp(hdr.value, "chunked") == 0)) { + parser->http.response->chunked_state = GRPC_HTTP_CHUNKED_LENGTH; + } break; case GRPC_HTTP_REQUEST: hdr_count = &parser->http.request->hdr_count; @@ -245,17 +253,24 @@ static grpc_error_handle finish_line(grpc_http_parser* parser, parser->state = GRPC_HTTP_HEADERS; break; case GRPC_HTTP_HEADERS: + case GRPC_HTTP_TRAILERS: if (parser->cur_line_length == parser->cur_line_end_length) { - parser->state = GRPC_HTTP_BODY; - *found_body_start = true; + if (parser->state == GRPC_HTTP_HEADERS) { + parser->state = GRPC_HTTP_BODY; + *found_body_start = true; + } else { + parser->state = GRPC_HTTP_END; + } break; - } - err = add_header(parser); - if (err != GRPC_ERROR_NONE) { - return err; + } else { + err = add_header(parser); + if (err != GRPC_ERROR_NONE) { + return err; + } } break; case GRPC_HTTP_BODY: + case GRPC_HTTP_END: GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Should never reach here")); } @@ -269,6 +284,59 @@ static grpc_error_handle addbyte_body(grpc_http_parser* parser, uint8_t byte) { char** body = nullptr; if (parser->type == GRPC_HTTP_RESPONSE) { + switch (parser->http.response->chunked_state) { + case GRPC_HTTP_CHUNKED_LENGTH: + if ((byte == '\r') || (byte == ';')) { + parser->http.response->chunked_state = + GRPC_HTTP_CHUNKED_IGNORE_ALL_UNTIL_LF; + } else if ((byte >= '0') && (byte <= '9')) { + parser->http.response->chunk_length *= 16; + parser->http.response->chunk_length += byte - '0'; + } else if ((byte >= 'a') && (byte <= 'f')) { + parser->http.response->chunk_length *= 16; + parser->http.response->chunk_length += byte - 'a' + 10; + } else if ((byte >= 'A') && (byte <= 'F')) { + parser->http.response->chunk_length *= 16; + parser->http.response->chunk_length += byte - 'A' + 10; + } else { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Expected chunk size in hexadecimal"); + } + return GRPC_ERROR_NONE; + case GRPC_HTTP_CHUNKED_IGNORE_ALL_UNTIL_LF: + if (byte == '\n') { + if (parser->http.response->chunk_length == 0) { + parser->state = GRPC_HTTP_TRAILERS; + } else { + parser->http.response->chunked_state = GRPC_HTTP_CHUNKED_BODY; + } + } + return GRPC_ERROR_NONE; + case GRPC_HTTP_CHUNKED_BODY: + if (parser->http.response->chunk_length == 0) { + if (byte != '\r') { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Expected '\\r\\n' after chunk body"); + } + parser->http.response->chunked_state = GRPC_HTTP_CHUNKED_CONSUME_LF; + parser->http.response->chunk_length = 0; + return GRPC_ERROR_NONE; + } else { + parser->http.response->chunk_length--; + /* fallback to the normal body appending code below */ + } + break; + case GRPC_HTTP_CHUNKED_CONSUME_LF: + if (byte != '\n') { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Expected '\\r\\n' after chunk body"); + } + parser->http.response->chunked_state = GRPC_HTTP_CHUNKED_LENGTH; + return GRPC_ERROR_NONE; + case GRPC_HTTP_CHUNKED_PLAIN: + /* avoiding warning; just fallback to normal codepath */ + break; + } body_length = &parser->http.response->body_length; body = &parser->http.response->body; } else if (parser->type == GRPC_HTTP_REQUEST) { @@ -318,6 +386,7 @@ static grpc_error_handle addbyte(grpc_http_parser* parser, uint8_t byte, switch (parser->state) { case GRPC_HTTP_FIRST_LINE: case GRPC_HTTP_HEADERS: + case GRPC_HTTP_TRAILERS: if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) { if (GRPC_TRACE_FLAG_ENABLED(grpc_http1_trace)) { gpr_log(GPR_ERROR, "HTTP header max line length (%d) exceeded", @@ -334,6 +403,8 @@ static grpc_error_handle addbyte(grpc_http_parser* parser, uint8_t byte, return GRPC_ERROR_NONE; case GRPC_HTTP_BODY: return addbyte_body(parser, byte); + case GRPC_HTTP_END: + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unexpected byte after end"); } GPR_UNREACHABLE_CODE(return GRPC_ERROR_NONE); } @@ -385,7 +456,7 @@ grpc_error_handle grpc_http_parser_parse(grpc_http_parser* parser, } grpc_error_handle grpc_http_parser_eof(grpc_http_parser* parser) { - if (parser->state != GRPC_HTTP_BODY) { + if ((parser->state != GRPC_HTTP_BODY) && (parser->state != GRPC_HTTP_END)) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Did not finish headers"); } return GRPC_ERROR_NONE; diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h index 80147584e6f..63f9c9fd121 100644 --- a/src/core/lib/http/parser.h +++ b/src/core/lib/http/parser.h @@ -38,9 +38,19 @@ typedef struct grpc_http_header { typedef enum { GRPC_HTTP_FIRST_LINE, GRPC_HTTP_HEADERS, - GRPC_HTTP_BODY + GRPC_HTTP_BODY, + GRPC_HTTP_TRAILERS, + GRPC_HTTP_END, } grpc_http_parser_state; +typedef enum { + GRPC_HTTP_CHUNKED_PLAIN, + GRPC_HTTP_CHUNKED_LENGTH, + GRPC_HTTP_CHUNKED_IGNORE_ALL_UNTIL_LF, + GRPC_HTTP_CHUNKED_BODY, + GRPC_HTTP_CHUNKED_CONSUME_LF, +} grpc_http_parser_chunked_state; + typedef enum { GRPC_HTTP_HTTP10, GRPC_HTTP_HTTP11, @@ -77,6 +87,9 @@ typedef struct grpc_http_response { grpc_http_header* hdrs = nullptr; /* Body: length and contents; contents are NOT null-terminated */ size_t body_length = 0; + /* State of the chunked parser. Only valid for the response. */ + grpc_http_parser_chunked_state chunked_state = GRPC_HTTP_CHUNKED_PLAIN; + size_t chunk_length = 0; char* body = nullptr; } grpc_http_response; diff --git a/test/core/http/format_request_test.cc b/test/core/http/format_request_test.cc index b9a099b6f74..1bb2d8694e9 100644 --- a/test/core/http/format_request_test.cc +++ b/test/core/http/format_request_test.cc @@ -38,7 +38,7 @@ static void test_format_get_request(void) { slice = grpc_httpcli_format_get_request(&req, host, "/index.html"); GPR_ASSERT(0 == grpc_slice_str_cmp(slice, - "GET /index.html HTTP/1.0\r\n" + "GET /index.html HTTP/1.1\r\n" "Host: example.com\r\n" "Connection: close\r\n" "User-Agent: " GRPC_HTTPCLI_USER_AGENT @@ -64,7 +64,7 @@ static void test_format_post_request(void) { slice = grpc_httpcli_format_post_request(&req, host, "/index.html"); GPR_ASSERT(0 == grpc_slice_str_cmp(slice, - "POST /index.html HTTP/1.0\r\n" + "POST /index.html HTTP/1.1\r\n" "Host: example.com\r\n" "Connection: close\r\n" "User-Agent: " GRPC_HTTPCLI_USER_AGENT @@ -91,7 +91,7 @@ static void test_format_post_request_no_body(void) { slice = grpc_httpcli_format_post_request(&req, host, "/index.html"); GPR_ASSERT(0 == grpc_slice_str_cmp(slice, - "POST /index.html HTTP/1.0\r\n" + "POST /index.html HTTP/1.1\r\n" "Host: example.com\r\n" "Connection: close\r\n" "User-Agent: " GRPC_HTTPCLI_USER_AGENT @@ -122,7 +122,7 @@ static void test_format_post_request_content_type_override(void) { GPR_ASSERT(0 == grpc_slice_str_cmp( slice, - "POST /index.html HTTP/1.0\r\n" + "POST /index.html HTTP/1.1\r\n" "Host: example.com\r\n" "Connection: close\r\n" "User-Agent: " GRPC_HTTPCLI_USER_AGENT "\r\n" diff --git a/test/core/http/parser_test.cc b/test/core/http/parser_test.cc index 6e31079d96f..1c63620e24c 100644 --- a/test/core/http/parser_test.cc +++ b/test/core/http/parser_test.cc @@ -245,6 +245,34 @@ int main(int argc, char** argv) { "\n" "abc", 200, "abc", NULL); + test_succeeds(split_modes[i], + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "4\r\n" + "This\r\n" + "16;param1;param2\r\n" + " is a chunked encoding\r\n" + "1D\r\n" + " example.\r\nNo params handled.\r\n" + "0\r\n" + "\r\n", + 200, + "This is a chunked encoding example.\r\nNo params handled.", + "Transfer-Encoding", "chunked", NULL); + test_succeeds(split_modes[i], + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "e\r\n" + "HTTP Trailers \r\n" + "13\r\n" + "are also supported.\r\n" + "0\r\n" + "abc: xyz\r\n" + "\r\n", + 200, "HTTP Trailers are also supported.", "Transfer-Encoding", + "chunked", "abc", "xyz", NULL); test_request_succeeds(split_modes[i], "GET / HTTP/1.0\r\n" "\r\n",