From 22a5c5ba09457c549381be98e635a918ef53cf0a Mon Sep 17 00:00:00 2001 From: "data-plane-api(CircleCI)" Date: Wed, 30 May 2018 23:38:54 +0000 Subject: [PATCH] access log service: cleanup and docs (#3494) Fixes https://github.com/envoyproxy/envoy/issues/743 This is a general cleanup of all of the access logging documentation. I have reorganized a bunch of things and hidden the various gRPC logging fields that are not implemented yet. I've also moved the existing tap protos into a new "output" directory. This is the best name I could come up for cleanly separating output data that might be stored outside of any service or configuration. Signed-off-by: Matt Klein Mirrored from https://github.com/envoyproxy/envoy @ c15019e79c832d9f0a09468affaadabc4be3e115 --- docs/BUILD | 6 +- envoy/api/v2/BUILD | 2 +- envoy/config/accesslog/v2/BUILD | 5 + envoy/config/accesslog/v2/als.proto | 39 ++- envoy/config/accesslog/v2/file.proto | 21 ++ envoy/config/filter/accesslog/v2/BUILD | 5 - .../filter/accesslog/v2/accesslog.proto | 307 +----------------- envoy/config/rbac/v2alpha/rbac.proto | 2 + .../capture/v2alpha/capture.proto | 4 +- envoy/data/accesslog/v2/BUILD | 24 ++ envoy/data/accesslog/v2/accesslog.proto | 288 ++++++++++++++++ .../common => data}/tap/v2alpha/BUILD | 0 .../common => data}/tap/v2alpha/capture.proto | 2 +- envoy/service/accesslog/v2/BUILD | 4 +- envoy/service/accesslog/v2/als.proto | 19 +- tools/BUILD | 2 +- tools/capture2pcap.py | 2 +- 17 files changed, 394 insertions(+), 338 deletions(-) create mode 100644 envoy/config/accesslog/v2/file.proto create mode 100644 envoy/data/accesslog/v2/BUILD create mode 100644 envoy/data/accesslog/v2/accesslog.proto rename envoy/{extensions/common => data}/tap/v2alpha/BUILD (100%) rename envoy/{extensions/common => data}/tap/v2alpha/capture.proto (97%) diff --git a/docs/BUILD b/docs/BUILD index 56e03ee8..82902b88 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -24,6 +24,8 @@ proto_library( "//envoy/api/v2/listener", "//envoy/api/v2/ratelimit", "//envoy/api/v2/route", + "//envoy/config/accesslog/v2:als", + "//envoy/config/accesslog/v2:file", "//envoy/config/bootstrap/v2:bootstrap", "//envoy/config/filter/accesslog/v2:accesslog", "//envoy/config/filter/http/buffer/v2:buffer", @@ -52,7 +54,9 @@ proto_library( "//envoy/config/rbac/v2alpha:rbac", "//envoy/config/trace/v2:trace", "//envoy/config/transport_socket/capture/v2alpha:capture", - "//envoy/extensions/common/tap/v2alpha:capture", + "//envoy/data/accesslog/v2:accesslog", + "//envoy/data/tap/v2alpha:capture", + "//envoy/service/accesslog/v2:als", "//envoy/service/discovery/v2:ads", "//envoy/service/load_stats/v2:lrs", "//envoy/service/metrics/v2:metrics_service", diff --git a/envoy/api/v2/BUILD b/envoy/api/v2/BUILD index ca664da9..92ce7a75 100644 --- a/envoy/api/v2/BUILD +++ b/envoy/api/v2/BUILD @@ -11,7 +11,7 @@ package_group( "//envoy/admin/...", "//envoy/api/v2", "//envoy/config/...", - "//envoy/extensions/...", + "//envoy/data/...", "//envoy/service/...", ], ) diff --git a/envoy/config/accesslog/v2/BUILD b/envoy/config/accesslog/v2/BUILD index 355fe3e9..63bdc5c5 100644 --- a/envoy/config/accesslog/v2/BUILD +++ b/envoy/config/accesslog/v2/BUILD @@ -9,3 +9,8 @@ api_proto_library( "//envoy/api/v2/core:grpc_service", ], ) + +api_proto_library( + name = "file", + srcs = ["file.proto"], +) diff --git a/envoy/config/accesslog/v2/als.proto b/envoy/config/accesslog/v2/als.proto index 92b6d365..5dd965a6 100644 --- a/envoy/config/accesslog/v2/als.proto +++ b/envoy/config/accesslog/v2/als.proto @@ -7,37 +7,40 @@ import "envoy/api/v2/core/grpc_service.proto"; import "validate/validate.proto"; -// Configuration for the built-in *envoy.tcp_grpc_access_log* type. This configuration will -// populate *StreamAccessLogsMessage.tcp_logs*. -// [#not-implemented-hide:] -// [#comment:TODO(mattklein123): Block type in non-tcp proxy cases?] -message TcpGrpcAccessLogConfig { - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message.required = true]; -} +// [#protodoc-title: gRPC Access Log Service (ALS)] -// Configuration for the built-in *envoy.http_grpc_access_log* type. This configuration will -// populate *StreamAccessLogsMessage.http_logs*. -// [#not-implemented-hide:] -// [#comment:TODO(mattklein123): Block type in non-http/router proxy cases?] +// Configuration for the built-in *envoy.http_grpc_access_log* +// :ref:`AccessLog `. This configuration will +// populate :ref:`StreamAccessLogsMessage.http_logs +// `. message HttpGrpcAccessLogConfig { CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message.required = true]; - // Additional request headers to log in *HTTPRequestProperties.request_headers*. + // Additional request headers to log in :ref:`HTTPRequestProperties.request_headers + // `. repeated string additional_request_headers_to_log = 2; - // Additional response headers to log in *HTTPResponseProperties.response_headers*. + // Additional response headers to log in :ref:`HTTPResponseProperties.response_headers + // `. repeated string additional_response_headers_to_log = 3; - // Additional response trailers to log in *HTTPResponseProperties.response_trailers*. + // Additional response trailers to log in :ref:`HTTPResponseProperties.response_trailers + // `. repeated string additional_response_trailers_to_log = 4; } -// Common configuration for gRPC access logs. +// Configuration for the built-in *envoy.tcp_grpc_access_log* type. This configuration will +// populate *StreamAccessLogsMessage.tcp_logs*. // [#not-implemented-hide:] +message TcpGrpcAccessLogConfig { + CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message.required = true]; +} + +// Common configuration for gRPC access logs. message CommonGrpcAccessLogConfig { - // The friendly name of the access log to be returned in StreamAccessLogsMessage.Identifier. This - // allows the access log server to differentiate between different access logs coming from the - // same Envoy. + // The friendly name of the access log to be returned in :ref:`StreamAccessLogsMessage.Identifier + // `. This allows the + // access log server to differentiate between different access logs coming from the same Envoy. string log_name = 1 [(validate.rules).string.min_bytes = 1]; // The gRPC service for the access log service. diff --git a/envoy/config/accesslog/v2/file.proto b/envoy/config/accesslog/v2/file.proto new file mode 100644 index 00000000..d1ca2d1e --- /dev/null +++ b/envoy/config/accesslog/v2/file.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.accesslog.v2; +option go_package = "v2"; + +import "validate/validate.proto"; + +// [#protodoc-title: File access log] + +// Custom configuration for an :ref:`AccessLog ` +// that writes log entries directly to a file. Configures the built-in *envoy.file_access_log* +// AccessLog. +message FileAccessLog { + // A path to a local file to which to write the access log entries. + string path = 1 [(validate.rules).string.min_bytes = 1]; + + // Access log format. Envoy supports :ref:`custom access log formats + // ` as well as a :ref:`default format + // `. + string format = 2; +} diff --git a/envoy/config/filter/accesslog/v2/BUILD b/envoy/config/filter/accesslog/v2/BUILD index 0097c1d6..fbab9f76 100644 --- a/envoy/config/filter/accesslog/v2/BUILD +++ b/envoy/config/filter/accesslog/v2/BUILD @@ -9,12 +9,9 @@ api_proto_library( "//envoy/config/filter/http/router/v2:__pkg__", "//envoy/config/filter/network/http_connection_manager/v2:__pkg__", "//envoy/config/filter/network/tcp_proxy/v2:__pkg__", - "//envoy/service/accesslog/v2:__pkg__", ], deps = [ - "//envoy/api/v2/core:address", "//envoy/api/v2/core:base", - "//envoy/api/v2/core:grpc_service", "//envoy/api/v2/route", "//envoy/type:percent", ], @@ -24,9 +21,7 @@ api_go_proto_library( name = "accesslog", proto = ":accesslog", deps = [ - "//envoy/api/v2/core:address_go_proto", "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:grpc_service_go_proto", "//envoy/api/v2/route:route_go_proto", "//envoy/type:percent_go_proto", ], diff --git a/envoy/config/filter/accesslog/v2/accesslog.proto b/envoy/config/filter/accesslog/v2/accesslog.proto index 378fb572..af78123e 100644 --- a/envoy/config/filter/accesslog/v2/accesslog.proto +++ b/envoy/config/filter/accesslog/v2/accesslog.proto @@ -3,311 +3,34 @@ syntax = "proto3"; package envoy.config.filter.accesslog.v2; option go_package = "v2"; -import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/route/route.proto"; import "envoy/type/percent.proto"; -import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; -import "google/protobuf/timestamp.proto"; -import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Common access log types] -// Envoy access logs describe incoming interaction with Envoy over a fixed -// period of time, and typically cover a single request/response exchange, -// (e.g. HTTP), stream (e.g. over HTTP/gRPC), or proxied connection (e.g. TCP). -// Access logs contain fields defined in protocol-specific protobuf messages. -// -// Except where explicitly declared otherwise, all fields describe -// *downstream* interaction between Envoy and a connected client. -// Fields describing *upstream* interaction will explicitly include ``upstream`` -// in their name. - -// Defines fields that are shared by all Envoy access logs. -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -message AccessLogCommon { - // This field indicates the rate at which this log entry was sampled. - // Valid range is (0.0, 1.0]. - double sample_rate = 1 [(validate.rules).double.gt = 0.0, (validate.rules).double.lte = 1.0]; - - // This field is the remote/origin address on which the request from the user was received. - // Note: This may not be the physical peer. E.g, if the remote address is inferred from for - // example the x-forwarder-for header, proxy protocol, etc. - envoy.api.v2.core.Address downstream_remote_address = 2; - - // This field is the local/destination address on which the request from the user was received. - envoy.api.v2.core.Address downstream_local_address = 3; - - // If the connection is secure, this field will contain TLS properties. - TLSProperties tls_properties = 4; - - // The time that Envoy started servicing this request. This is effectively the time that the first - // downstream byte is received. - google.protobuf.Timestamp start_time = 5 [(gogoproto.stdtime) = true]; - - // Interval between the first downstream byte received and the last - // downstream byte received (i.e. time it takes to receive a request). - google.protobuf.Duration time_to_last_rx_byte = 6 [(gogoproto.stdduration) = true]; - - // Interval between the first downstream byte received and the first upstream byte sent. There may - // by considerable delta between *time_to_last_rx_byte* and this value due to filters. - // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about - // not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_first_upstream_tx_byte = 7 [(gogoproto.stdduration) = true]; - - // Interval between the first downstream byte received and the last upstream byte sent. There may - // by considerable delta between *time_to_last_rx_byte* and this value due to filters. - // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about - // not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_last_upstream_tx_byte = 8 [(gogoproto.stdduration) = true]; - - // Interval between the first downstream byte received and the first upstream - // byte received (i.e. time it takes to start receiving a response). - google.protobuf.Duration time_to_first_upstream_rx_byte = 9 [(gogoproto.stdduration) = true]; - - // Interval between the first downstream byte received and the last upstream - // byte received (i.e. time it takes to receive a complete response). - google.protobuf.Duration time_to_last_upstream_rx_byte = 10 [(gogoproto.stdduration) = true]; - - // Interval between the first downstream byte received and the first downstream byte sent. - // There may be a considerable delta between the *time_to_first_upstream_rx_byte* and this field - // due to filters. Additionally, the same caveats apply as documented in - // *time_to_last_downstream_tx_byte* about not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_first_downstream_tx_byte = 11 [(gogoproto.stdduration) = true]; - - // Interval between the first downstream byte received and the last downstream byte sent. - // Depending on protocol, buffering, windowing, filters, etc. there may be a considerable delta - // between *time_to_last_upstream_rx_byte* and this field. Note also that this is an approximate - // time. In the current implementation it does not include kernel socket buffer time. In the - // current implementation it also does not include send window buffering inside the HTTP/2 codec. - // In the future it is likely that work will be done to make this duration more accurate. - google.protobuf.Duration time_to_last_downstream_tx_byte = 12 [(gogoproto.stdduration) = true]; - - // The upstream remote/destination address that handles this exchange. This does not include - // retries. - envoy.api.v2.core.Address upstream_remote_address = 13; - - // The upstream local/origin address that handles this exchange. This does not include retries. - envoy.api.v2.core.Address upstream_local_address = 14; - - // The upstream cluster that *upstream_remote_address* belongs to. - string upstream_cluster = 15; - - // Flags indicating occurrences during request/response processing. - ResponseFlags response_flags = 16; - - // All metadata encountered during request processing, including endpoint - // selection. - // - // This can be used to associate IDs attached to the various configurations - // used to process this request with the access log entry. For example, a - // route created from a higher level forwarding rule with some ID can place - // that ID in this field and cross reference later. It can also be used to - // determine if a canary endpoint was used or not. - envoy.api.v2.core.Metadata metadata = 17; -} - -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -// Flags indicating occurrences during request/response processing. -message ResponseFlags { - // Indicates local server healthcheck failed. - bool failed_local_healthcheck = 1; - - // Indicates there was no healthy upstream. - bool no_healthy_upstream = 2; - - // Indicates an there was an upstream request timeout. - bool upstream_request_timeout = 3; - - // Indicates local codec level reset was sent on the stream. - bool local_reset = 4; - - // Indicates remote codec level reset was received on the stream. - bool upstream_remote_reset = 5; - - // Indicates there was a local reset by a connection pool due to an initial connection failure. - bool upstream_connection_failure = 6; - - // Indicates the stream was reset locally due to connection termination. - bool upstream_connection_termination = 7; - - // Indicates the stream was reset because of a resource overflow. - bool upstream_overflow = 8; - - // Indicates no route was found for the request. - bool no_route_found = 9; - - // Indicates that the request was delayed before proxying. - bool delay_injected = 10; - - // Indicates that the request was aborted with an injected error code. - bool fault_injected = 11; - - // Indicates that the request was rate-limited locally. - bool rate_limited = 12; - - message Unauthorized { - // Reasons why the request was unauthorized - enum Reason { - REASON_UNSPECIFIED = 0; - // The request was denied by the external authorization service. - EXTERNAL_SERVICE = 1; - } - - Reason reason = 1; - } - - // Indicates if the request was deemed unauthorized and the reason for it. - Unauthorized unauthorized_details = 13; -} - -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -// Properties of a negotiated TLS connection. -message TLSProperties { - enum TLSVersion { - VERSION_UNSPECIFIED = 0; - TLSv1 = 1; - TLSv1_1 = 2; - TLSv1_2 = 3; - TLSv1_3 = 4; - } - // Version of TLS that was negotiated. - TLSVersion tls_version = 1; - - // TLS cipher suite negotiated during handshake. The value is a - // four-digit hex code defined by the IANA TLS Cipher Suite Registry - // (e.g. ``009C`` for ``TLS_RSA_WITH_AES_128_GCM_SHA256``). - // - // Here it is expressed as an integer. - google.protobuf.UInt32Value tls_cipher_suite = 2; - - // SNI hostname from handshake. - string tls_sni_hostname = 3; -} - -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -message TCPAccessLogEntry { - // Common properties shared by all Envoy access logs. - AccessLogCommon common_properties = 1; -} - -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -message HTTPRequestProperties { - // The request method (RFC 7231/2616). - // [#comment:TODO(htuch): add (validate.rules).enum.defined_only = true once - // https://github.com/lyft/protoc-gen-validate/issues/42 is resolved.] - envoy.api.v2.core.RequestMethod request_method = 1; - - // The scheme portion of the incoming request URI. - string scheme = 2; - - // HTTP/2 ``:authority`` or HTTP/1.1 ``Host`` header value. - string authority = 3; - - // The port of the incoming request URI - // (unused currently, as port is composed onto authority). - google.protobuf.UInt32Value port = 4; - - // The path portion from the incoming request URI. - string path = 5; - - // Value of the ``User-Agent`` request header. - string user_agent = 6; - - // Value of the ``Referer`` request header. - string referer = 7; - - // Value of the ``X-Forwarded-For`` request header. - string forwarded_for = 8; - - // Value of the ``X-Request-Id`` request header - // - // This header is used by Envoy to uniquely identify a request. - // It will be generated for all external requests and internal requests that - // do not already have a request ID. - string request_id = 9; - - // Value of the ``X-Envoy-Original-Path`` request header. - string original_path = 10; - - // Size of the HTTP request headers in bytes. - // - // This value is captured from the OSI layer 7 perspective, i.e. it does not - // include overhead from framing or encoding at other networking layers. - uint64 request_headers_bytes = 11; - - // Size of the HTTP request body in bytes. - // - // This value is captured from the OSI layer 7 perspective, i.e. it does not - // include overhead from framing or encoding at other networking layers. - uint64 request_body_bytes = 12; - - // Map of additional headers that have been configured to be logged. - map request_headers = 13; -} - -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -message HTTPResponseProperties { - // The HTTP response code returned by Envoy. - google.protobuf.UInt32Value response_code = 1; - - // Size of the HTTP response headers in bytes. - // - // This value is captured from the OSI layer 7 perspective, i.e. it does not - // include overhead from framing or encoding at other networking layers. - uint64 response_headers_bytes = 2; - - // Size of the HTTP response body in bytes. - // - // This value is captured from the OSI layer 7 perspective, i.e. it does not - // include overhead from framing or encoding at other networking layers. - uint64 response_body_bytes = 3; - - // Map of additional headers configured to be logged. - map response_headers = 4; - - // Map of trailers configured to be logged. - map response_trailers = 5; -} - -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. -message HTTPAccessLogEntry { - // Common properties shared by all Envoy access logs. - AccessLogCommon common_properties = 1; - - // HTTP version - enum HTTPVersion { - PROTOCOL_UNSPECIFIED = 0; - HTTP10 = 1; - HTTP11 = 2; - HTTP2 = 3; - } - HTTPVersion protocol_version = 2; - - // Description of the incoming HTTP request. - HTTPRequestProperties request = 3; - - // Description of the outgoing HTTP response. - HTTPResponseProperties response = 4; -} message AccessLog { // The name of the access log implementation to instantiate. The name must // match a statically registered access log. Current built-in loggers include: - // 1) "envoy.file_access_log" + // + // #. "envoy.file_access_log" + // #. "envoy.http_grpc_access_log" string name = 1; // Filter which is used to determine if the access log needs to be written. AccessLogFilter filter = 2; - // Custom configuration that depends on the access log being instantiated. built-in configurations + // Custom configuration that depends on the access log being instantiated. Built-in configurations // include: - // 1) "envoy.file_access_log": :ref:`FileAccessLog - // ` + // + // #. "envoy.file_access_log": :ref:`FileAccessLog + // ` + // #. "envoy.http_grpc_access_log": :ref:`HttpGrpcAccessLogConfig + // ` google.protobuf.Struct config = 3; } @@ -427,15 +150,3 @@ message HeaderFilter { // check. envoy.api.v2.route.HeaderMatcher header = 1 [(validate.rules).message.required = true]; } - -// Custom configuration for an AccessLog that writes log entries directly to a file. -// Configures the built-in *envoy.file_access_log* AccessLog. -message FileAccessLog { - // A path to a local file to which to write the access log entries. - string path = 1 [(validate.rules).string.min_bytes = 1]; - - // Access log format. Envoy supports :ref:`custom access log formats - // ` as well as a :ref:`default format - // `. - string format = 2; -} diff --git a/envoy/config/rbac/v2alpha/rbac.proto b/envoy/config/rbac/v2alpha/rbac.proto index 818054ad..cb9e53b5 100644 --- a/envoy/config/rbac/v2alpha/rbac.proto +++ b/envoy/config/rbac/v2alpha/rbac.proto @@ -7,6 +7,8 @@ import "envoy/api/v2/route/route.proto"; package envoy.config.rbac.v2alpha; option go_package = "v2alpha"; +// [#protodoc-title: Role Based Access Control (RBAC)] + // Role Based Access Control (RBAC) provides service-level and method-level access control for a // service. RBAC policies are additive. The policies are examined in order. A request is allowed // once a matching policy is found (suppose the `action` is ALLOW). diff --git a/envoy/config/transport_socket/capture/v2alpha/capture.proto b/envoy/config/transport_socket/capture/v2alpha/capture.proto index 07b2802d..6d2f9a0e 100644 --- a/envoy/config/transport_socket/capture/v2alpha/capture.proto +++ b/envoy/config/transport_socket/capture/v2alpha/capture.proto @@ -23,10 +23,10 @@ message FileSink { // File format. enum Format { // Binary proto format as per :ref:`Trace - // `. + // `. PROTO_BINARY = 0; // Text proto format as per :ref:`Trace - // `. + // `. PROTO_TEXT = 1; } Format format = 2; diff --git a/envoy/data/accesslog/v2/BUILD b/envoy/data/accesslog/v2/BUILD new file mode 100644 index 00000000..21c1ea44 --- /dev/null +++ b/envoy/data/accesslog/v2/BUILD @@ -0,0 +1,24 @@ +load("//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "accesslog", + srcs = ["accesslog.proto"], + visibility = [ + "//envoy/service/accesslog/v2:__pkg__", + ], + deps = [ + "//envoy/api/v2/core:address", + "//envoy/api/v2/core:base", + ], +) + +api_go_proto_library( + name = "accesslog", + proto = ":accesslog", + deps = [ + "//envoy/api/v2/core:address_go_proto", + "//envoy/api/v2/core:base_go_proto", + ], +) diff --git a/envoy/data/accesslog/v2/accesslog.proto b/envoy/data/accesslog/v2/accesslog.proto new file mode 100644 index 00000000..129e996d --- /dev/null +++ b/envoy/data/accesslog/v2/accesslog.proto @@ -0,0 +1,288 @@ +syntax = "proto3"; + +package envoy.data.accesslog.v2; + +import "envoy/api/v2/core/address.proto"; +import "envoy/api/v2/core/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; +import "gogoproto/gogo.proto"; +import "validate/validate.proto"; + +// [#protodoc-title: gRPC access logs] +// Envoy access logs describe incoming interaction with Envoy over a fixed +// period of time, and typically cover a single request/response exchange, +// (e.g. HTTP), stream (e.g. over HTTP/gRPC), or proxied connection (e.g. TCP). +// Access logs contain fields defined in protocol-specific protobuf messages. +// +// Except where explicitly declared otherwise, all fields describe +// *downstream* interaction between Envoy and a connected client. +// Fields describing *upstream* interaction will explicitly include ``upstream`` +// in their name. + +// [#not-implemented-hide:] +message TCPAccessLogEntry { + // Common properties shared by all Envoy access logs. + AccessLogCommon common_properties = 1; +} + +message HTTPAccessLogEntry { + // Common properties shared by all Envoy access logs. + AccessLogCommon common_properties = 1; + + // HTTP version + enum HTTPVersion { + PROTOCOL_UNSPECIFIED = 0; + HTTP10 = 1; + HTTP11 = 2; + HTTP2 = 3; + } + HTTPVersion protocol_version = 2; + + // Description of the incoming HTTP request. + HTTPRequestProperties request = 3; + + // Description of the outgoing HTTP response. + HTTPResponseProperties response = 4; +} + +// Defines fields that are shared by all Envoy access logs. +message AccessLogCommon { + // [#not-implemented-hide:] + // This field indicates the rate at which this log entry was sampled. + // Valid range is (0.0, 1.0]. + double sample_rate = 1 [(validate.rules).double.gt = 0.0, (validate.rules).double.lte = 1.0]; + + // This field is the remote/origin address on which the request from the user was received. + // Note: This may not be the physical peer. E.g, if the remote address is inferred from for + // example the x-forwarder-for header, proxy protocol, etc. + envoy.api.v2.core.Address downstream_remote_address = 2; + + // This field is the local/destination address on which the request from the user was received. + envoy.api.v2.core.Address downstream_local_address = 3; + + // [#not-implemented-hide:] + // If the connection is secure,S this field will contain TLS properties. + TLSProperties tls_properties = 4; + + // The time that Envoy started servicing this request. This is effectively the time that the first + // downstream byte is received. + google.protobuf.Timestamp start_time = 5 [(gogoproto.stdtime) = true]; + + // Interval between the first downstream byte received and the last + // downstream byte received (i.e. time it takes to receive a request). + google.protobuf.Duration time_to_last_rx_byte = 6 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the first upstream byte sent. There may + // by considerable delta between *time_to_last_rx_byte* and this value due to filters. + // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about + // not accounting for kernel socket buffer time, etc. + google.protobuf.Duration time_to_first_upstream_tx_byte = 7 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the last upstream byte sent. There may + // by considerable delta between *time_to_last_rx_byte* and this value due to filters. + // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about + // not accounting for kernel socket buffer time, etc. + google.protobuf.Duration time_to_last_upstream_tx_byte = 8 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the first upstream + // byte received (i.e. time it takes to start receiving a response). + google.protobuf.Duration time_to_first_upstream_rx_byte = 9 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the last upstream + // byte received (i.e. time it takes to receive a complete response). + google.protobuf.Duration time_to_last_upstream_rx_byte = 10 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the first downstream byte sent. + // There may be a considerable delta between the *time_to_first_upstream_rx_byte* and this field + // due to filters. Additionally, the same caveats apply as documented in + // *time_to_last_downstream_tx_byte* about not accounting for kernel socket buffer time, etc. + google.protobuf.Duration time_to_first_downstream_tx_byte = 11 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the last downstream byte sent. + // Depending on protocol, buffering, windowing, filters, etc. there may be a considerable delta + // between *time_to_last_upstream_rx_byte* and this field. Note also that this is an approximate + // time. In the current implementation it does not include kernel socket buffer time. In the + // current implementation it also does not include send window buffering inside the HTTP/2 codec. + // In the future it is likely that work will be done to make this duration more accurate. + google.protobuf.Duration time_to_last_downstream_tx_byte = 12 [(gogoproto.stdduration) = true]; + + // The upstream remote/destination address that handles this exchange. This does not include + // retries. + envoy.api.v2.core.Address upstream_remote_address = 13; + + // The upstream local/origin address that handles this exchange. This does not include retries. + envoy.api.v2.core.Address upstream_local_address = 14; + + // The upstream cluster that *upstream_remote_address* belongs to. + string upstream_cluster = 15; + + // Flags indicating occurrences during request/response processing. + ResponseFlags response_flags = 16; + + // All metadata encountered during request processing, including endpoint + // selection. + // + // This can be used to associate IDs attached to the various configurations + // used to process this request with the access log entry. For example, a + // route created from a higher level forwarding rule with some ID can place + // that ID in this field and cross reference later. It can also be used to + // determine if a canary endpoint was used or not. + envoy.api.v2.core.Metadata metadata = 17; +} + +// Flags indicating occurrences during request/response processing. +message ResponseFlags { + // Indicates local server healthcheck failed. + bool failed_local_healthcheck = 1; + + // Indicates there was no healthy upstream. + bool no_healthy_upstream = 2; + + // Indicates an there was an upstream request timeout. + bool upstream_request_timeout = 3; + + // Indicates local codec level reset was sent on the stream. + bool local_reset = 4; + + // Indicates remote codec level reset was received on the stream. + bool upstream_remote_reset = 5; + + // Indicates there was a local reset by a connection pool due to an initial connection failure. + bool upstream_connection_failure = 6; + + // Indicates the stream was reset locally due to connection termination. + bool upstream_connection_termination = 7; + + // Indicates the stream was reset because of a resource overflow. + bool upstream_overflow = 8; + + // Indicates no route was found for the request. + bool no_route_found = 9; + + // Indicates that the request was delayed before proxying. + bool delay_injected = 10; + + // Indicates that the request was aborted with an injected error code. + bool fault_injected = 11; + + // Indicates that the request was rate-limited locally. + bool rate_limited = 12; + + message Unauthorized { + // Reasons why the request was unauthorized + enum Reason { + REASON_UNSPECIFIED = 0; + // The request was denied by the external authorization service. + EXTERNAL_SERVICE = 1; + } + + Reason reason = 1; + } + + // Indicates if the request was deemed unauthorized and the reason for it. + Unauthorized unauthorized_details = 13; +} + +// [#not-implemented-hide:] +// Properties of a negotiated TLS connection. +message TLSProperties { + enum TLSVersion { + VERSION_UNSPECIFIED = 0; + TLSv1 = 1; + TLSv1_1 = 2; + TLSv1_2 = 3; + TLSv1_3 = 4; + } + // Version of TLS that was negotiated. + TLSVersion tls_version = 1; + + // TLS cipher suite negotiated during handshake. The value is a + // four-digit hex code defined by the IANA TLS Cipher Suite Registry + // (e.g. ``009C`` for ``TLS_RSA_WITH_AES_128_GCM_SHA256``). + // + // Here it is expressed as an integer. + google.protobuf.UInt32Value tls_cipher_suite = 2; + + // SNI hostname from handshake. + string tls_sni_hostname = 3; +} + +message HTTPRequestProperties { + // The request method (RFC 7231/2616). + // [#comment:TODO(htuch): add (validate.rules).enum.defined_only = true once + // https://github.com/lyft/protoc-gen-validate/issues/42 is resolved.] + envoy.api.v2.core.RequestMethod request_method = 1; + + // The scheme portion of the incoming request URI. + string scheme = 2; + + // HTTP/2 ``:authority`` or HTTP/1.1 ``Host`` header value. + string authority = 3; + + // The port of the incoming request URI + // (unused currently, as port is composed onto authority). + google.protobuf.UInt32Value port = 4; + + // The path portion from the incoming request URI. + string path = 5; + + // Value of the ``User-Agent`` request header. + string user_agent = 6; + + // Value of the ``Referer`` request header. + string referer = 7; + + // Value of the ``X-Forwarded-For`` request header. + string forwarded_for = 8; + + // Value of the ``X-Request-Id`` request header + // + // This header is used by Envoy to uniquely identify a request. + // It will be generated for all external requests and internal requests that + // do not already have a request ID. + string request_id = 9; + + // Value of the ``X-Envoy-Original-Path`` request header. + string original_path = 10; + + // Size of the HTTP request headers in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 request_headers_bytes = 11; + + // Size of the HTTP request body in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 request_body_bytes = 12; + + // Map of additional headers that have been configured to be logged. + map request_headers = 13; +} + +message HTTPResponseProperties { + // The HTTP response code returned by Envoy. + google.protobuf.UInt32Value response_code = 1; + + // Size of the HTTP response headers in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 response_headers_bytes = 2; + + // Size of the HTTP response body in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 response_body_bytes = 3; + + // Map of additional headers configured to be logged. + map response_headers = 4; + + // Map of trailers configured to be logged. + map response_trailers = 5; +} diff --git a/envoy/extensions/common/tap/v2alpha/BUILD b/envoy/data/tap/v2alpha/BUILD similarity index 100% rename from envoy/extensions/common/tap/v2alpha/BUILD rename to envoy/data/tap/v2alpha/BUILD diff --git a/envoy/extensions/common/tap/v2alpha/capture.proto b/envoy/data/tap/v2alpha/capture.proto similarity index 97% rename from envoy/extensions/common/tap/v2alpha/capture.proto rename to envoy/data/tap/v2alpha/capture.proto index cde6791b..57c68277 100644 --- a/envoy/extensions/common/tap/v2alpha/capture.proto +++ b/envoy/data/tap/v2alpha/capture.proto @@ -4,7 +4,7 @@ syntax = "proto3"; // Trace capture format for the capture transport socket extension. This dumps plain text read/write // sequences on a socket. -package envoy.extensions.common.tap.v2alpha; +package envoy.data.tap.v2alpha; option go_package = "v2"; import "envoy/api/v2/core/address.proto"; diff --git a/envoy/service/accesslog/v2/BUILD b/envoy/service/accesslog/v2/BUILD index 58c28619..c5073996 100644 --- a/envoy/service/accesslog/v2/BUILD +++ b/envoy/service/accesslog/v2/BUILD @@ -9,7 +9,7 @@ api_proto_library( deps = [ "//envoy/api/v2/core:base", "//envoy/api/v2/core:grpc_service", - "//envoy/config/filter/accesslog/v2:accesslog", + "//envoy/data/accesslog/v2:accesslog", ], ) @@ -18,6 +18,6 @@ api_go_grpc_library( proto = ":als", deps = [ "//envoy/api/v2/core:base_go_proto", - "//envoy/config/filter/accesslog/v2:accesslog_go_proto", + "//envoy/data/accesslog/v2:accesslog_go_proto", ], ) diff --git a/envoy/service/accesslog/v2/als.proto b/envoy/service/accesslog/v2/als.proto index ec60b6d1..168293e0 100644 --- a/envoy/service/accesslog/v2/als.proto +++ b/envoy/service/accesslog/v2/als.proto @@ -5,10 +5,12 @@ option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; -import "envoy/config/filter/accesslog/v2/accesslog.proto"; +import "envoy/data/accesslog/v2/accesslog.proto"; import "validate/validate.proto"; +// [#protodoc-title: gRPC Access Log Service (ALS)] + // Service for streaming access logs from Envoy to an access log server. service AccessLogService { // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any @@ -22,12 +24,9 @@ service AccessLogService { } // Empty response for the StreamAccessLogs API. Will never be sent. See below. -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message StreamAccessLogsResponse { } -// [#proto-status: experimental] -// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream // access logs without ever expecting a response. message StreamAccessLogsMessage { @@ -35,7 +34,8 @@ message StreamAccessLogsMessage { // The node sending the access log messages over the stream. envoy.api.v2.core.Node node = 1 [(validate.rules).message.required = true]; - // The friendly name of the log configured in AccessLogServiceConfig. + // The friendly name of the log configured in :ref:`CommonGrpcAccessLogConfig + // `. string log_name = 2 [(validate.rules).string.min_bytes = 1]; } @@ -45,22 +45,25 @@ message StreamAccessLogsMessage { // Wrapper for batches of HTTP access log entries. message HTTPAccessLogEntries { - repeated envoy.config.filter.accesslog.v2.HTTPAccessLogEntry log_entry = 1 + repeated envoy.data.accesslog.v2.HTTPAccessLogEntry log_entry = 1 [(validate.rules).repeated .min_items = 1]; } + // [#not-implemented-hide:] // Wrapper for batches of TCP access log entries. message TCPAccessLogEntries { - repeated envoy.config.filter.accesslog.v2.TCPAccessLogEntry log_entry = 1 + repeated envoy.data.accesslog.v2.TCPAccessLogEntry log_entry = 1 [(validate.rules).repeated .min_items = 1]; } // Batches of log entries of a single type. Generally speaking, a given stream should only - // ever incude one type of log entry. + // ever include one type of log entry. oneof log_entries { option (validate.required) = true; HTTPAccessLogEntries http_logs = 2; + + // [#not-implemented-hide:] TCPAccessLogEntries tcp_logs = 3; } } diff --git a/tools/BUILD b/tools/BUILD index a3336f19..7a86e4da 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -6,7 +6,7 @@ py_binary( licenses = ["notice"], # Apache 2 visibility = ["//visibility:public"], deps = [ - "//envoy/extensions/common/tap/v2alpha:capture_py", + "//envoy/data/tap/v2alpha:capture_py", ], ) diff --git a/tools/capture2pcap.py b/tools/capture2pcap.py index 52770b9f..efce87cc 100644 --- a/tools/capture2pcap.py +++ b/tools/capture2pcap.py @@ -27,7 +27,7 @@ import time from google.protobuf import text_format -from envoy.extensions.common.tap.v2alpha import capture_pb2 +from envoy.data.tap.v2alpha import capture_pb2 def DumpEvent(direction, timestamp, data):