From b227517de0e37a824277f60bed917a252d66128f Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 30 Nov 2017 12:15:45 -0800 Subject: [PATCH] docs: access log and misc fixes (#301) Signed-off-by: Matt Klein --- api/bootstrap.proto | 5 +- api/eds.proto | 5 + api/filter/accesslog/accesslog.proto | 86 +++++---- docs/root/api-v1/access_log.rst | 181 ++++++++++++++++++ docs/root/api-v1/api.rst | 1 + docs/root/configuration/access_log.rst | 176 +---------------- .../configuration/http_conn_man/headers.rst | 3 +- docs/root/intro/version_history.rst | 2 +- 8 files changed, 248 insertions(+), 211 deletions(-) create mode 100644 docs/root/api-v1/access_log.rst diff --git a/api/bootstrap.proto b/api/bootstrap.proto index 93248d3b..3c8d8225 100644 --- a/api/bootstrap.proto +++ b/api/bootstrap.proto @@ -103,9 +103,8 @@ message ClusterManager { BindConfig upstream_bind_config = 3; // A management server endpoint to stream load stats to via - // :ref:`StreamLoadStats `. This must have - // :ref:`api_type ` :ref:`GRPC - // `. + // *StreamLoadStats*. This must have :ref:`api_type ` + // :ref:`GRPC `. ApiConfigSource load_stats_config = 4; } diff --git a/api/eds.proto b/api/eds.proto index e5cf80f7..c1158f08 100644 --- a/api/eds.proto +++ b/api/eds.proto @@ -137,6 +137,7 @@ message LocalityLbEndpoints { uint32 priority = 5; } +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message EndpointLoadMetricStats { // Name of the metric; may be empty. string metric_name = 1; @@ -153,6 +154,7 @@ message EndpointLoadMetricStats { // defined by // :ref:`LoadStatsResponse.load_reporting_interval`. // Stats per upstream region/zone and optionally per subzone. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message UpstreamLocalityStats { // Name of zone, region and optionally endpoint group these metrics were // collected from. Zone and region names could be empty if unknown. @@ -199,6 +201,7 @@ message UpstreamLocalityStats { // Per cluster load stats. Envoy reports these stats a management server in a // :ref:`LoadStatsRequest` +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message ClusterStats { // The name of the cluster. string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; @@ -222,6 +225,7 @@ message ClusterStats { } // A load report Envoy sends to the management server. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message LoadStatsRequest { // Node identifier for Envoy instance. Node node = 1; @@ -264,6 +268,7 @@ message ClusterLoadAssignment { // The management server sends envoy a LoadStatsResponse with all clusters it // is interested in learning load stats about. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message LoadStatsResponse { // Clusters to report stats for. repeated string clusters = 1 [(validate.rules).repeated.min_items = 1]; diff --git a/api/filter/accesslog/accesslog.proto b/api/filter/accesslog/accesslog.proto index 8a3a365c..b0525f69 100644 --- a/api/filter/accesslog/accesslog.proto +++ b/api/filter/accesslog/accesslog.proto @@ -13,7 +13,6 @@ import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; // [#protodoc-title: Common access log types] -// [#proto-status: draft] // Envoy access logs describe incoming interaction with Envoy over a fixed // period of time, and typically cover a single request/response exchange, // (e.g. HTTP), stream (e.g. over HTTP/gRPC), or proxied connection (e.g. TCP). @@ -25,6 +24,8 @@ import "validate/validate.proto"; // in their name. // Defines fields that are shared by all Envoy access logs. +// [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message AccessLogCommon { // This field indicates the rate at which this log entry was sampled. // Valid range is (0.0, 1.0]. @@ -73,6 +74,7 @@ message AccessLogCommon { } // [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. // Flags indicating occurences during request/response processing. message ResponseFlags { // Indicates local server healthcheck failed. @@ -113,6 +115,7 @@ message ResponseFlags { } // [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. // Properties of a negotiated TLS connection. message TLSProperties { enum TLSVersion { @@ -137,12 +140,14 @@ message TLSProperties { } // [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message TCPAccessLogEntry { // Common properties shared by all Envoy access logs. AccessLogCommon common_properties = 1; } // [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message HTTPRequestProperties { // The request method (RFC 7231/2616). RequestMethod request_method = 1; @@ -196,6 +201,7 @@ message HTTPRequestProperties { } // [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message HTTPResponseProperties { // The HTTP response code returned by Envoy. google.protobuf.UInt32Value response_code = 1; @@ -217,6 +223,7 @@ message HTTPResponseProperties { } // [#proto-status: draft] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message HTTPAccessLogEntry { // Common properties shared by all Envoy access logs. AccessLogCommon common_properties = 1; @@ -237,14 +244,18 @@ message HTTPAccessLogEntry { HTTPResponseProperties response = 4; } -// Filter on some integer comparison. +// Filter on an integer comparison. message ComparisonFilter { - // Comparison operator. enum Op { + // = EQ = 0; + + // >= GE = 1; } - Op op = 1; + + // Comparison operator. + Op op = 1 [(validate.rules).enum.defined_only = true]; // Value to compare against. RuntimeUInt32 value = 2; @@ -252,80 +263,91 @@ message ComparisonFilter { // Filters on HTTP response/status code. message StatusCodeFilter { - ComparisonFilter comparison = 1; + // Comparison. + ComparisonFilter comparison = 1 [(validate.rules).message.required = true]; } // Filters on total request duration in milliseconds. message DurationFilter { - ComparisonFilter comparison = 1; + // Comparison. + ComparisonFilter comparison = 1 [(validate.rules).message.required = true]; } // Filters for requests that are not health check requests. A health check // request is marked by the health check filter. -message NotHealthCheckFilter { -} +message NotHealthCheckFilter {} // Filters for requests that are traceable. See the tracing overview for more // information on how a request becomes traceable. -message TraceableFilter { -} +message TraceableFilter {} // Filters for random sampling of requests. Sampling pivots on the header -// x-request-id being present. If x-request-id is present, the filter will -// consistently sample across multiple hosts based on the runtime key value and -// the value extracted from x-request-id. If it is missing, the filter will -// randomly sample based on the runtime key value. +// :ref:`x-request-id` being present. If +// :ref:`x-request-id` is present, the filter will +// consistently sample across multiple hosts based on the runtime key value and the value extracted +// from :ref:`x-request-id`. If it is missing, the +// filter will randomly sample based on the runtime key value. message RuntimeFilter { // Runtime key to get the percentage of requests to be sampled. This runtime // control is specified in the range 0-100 and defaults to 0. - string runtime_key = 1; + string runtime_key = 1 [(validate.rules).string.min_bytes = 1]; } // Performs a logical “and” operation on the result of each filter in filters. // Filters are evaluated sequentially and if one of them returns false, the // filter returns false immediately. message AndFilter { - repeated AccessLogFilter filters = 1; + repeated AccessLogFilter filters = 1 [(validate.rules).repeated.min_items = 2]; } // Performs a logical “or” operation on the result of each individual filter. // Filters are evaluated sequentially and if one of them returns true, the // filter returns true immediately. message OrFilter { - repeated AccessLogFilter filters = 2; + repeated AccessLogFilter filters = 2 [(validate.rules).repeated.min_items = 2]; } message AccessLogFilter { oneof filter_specifier { + option (validate.required) = true; + + // Status code filter. StatusCodeFilter status_code_filter = 1; + + // Duration filter. DurationFilter duration_filter = 2; + + // Not health check filter. NotHealthCheckFilter not_health_check_filter = 3; + + // Traceable filter. TraceableFilter traceable_filter = 4; + + // Runtime filter. RuntimeFilter runtime_filter = 5; + + // And filter. AndFilter and_filter = 6; + + // Or filter. OrFilter or_filter = 7; } } // Custom configuration for an AccessLog that writes log entries directly to a file. -// Configures the built-in "envoy.file_access_log" AccessLog. +// Configures the built-in *envoy.file_access_log* AccessLog. message FileAccessLog { // A path to a local file to which to write the access log entries. - string path = 1; - - // [V2-API-DIFF] In addition to the existing format specifiers, the following - // additional specifiers will be available: - // %TLS_SNI_HOSTNAME%: SNI from handshake. - // %TLS_VERSION%: Possible values include: “TLSv1”, “TLSv1.1”, “TLSv1.2”, “TLSv1.3”. - // %TLS_CIPHER_SUITE%: Cipher suite negotiated during the TLS handshake. The - // value is four hex digits defined by the IANA TLS Cipher Suite Registry, - // e.g. “009C” for TLS_RSA_WITH_AES_128_GCM_SHA256. - // TODO(htuch): Document how envoy.http_connection_manager.access_log values - // can be accessed in the format specifier. + string path = 1 [(validate.rules).string.min_bytes = 1]; + + // Access log format. Envoy supports :ref:`custom access log formats + // ` as well as a :ref:`default format + // `. string format = 2; } // Configuration for the built-in "envoy.grpc_access_log" access log type. +// [#not-implemented-hide:] message AccessLogServiceConfig { // The friendly name of the access log to be returned in StreamAccessLogsMessage.Identifier. This // allows the access log server to differentiate between different access logs coming from the @@ -338,6 +360,7 @@ message AccessLogServiceConfig { } // [#proto-status: experimental] +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream // access logs without ever expecting a response. message StreamAccessLogsMessage { @@ -372,6 +395,7 @@ message StreamAccessLogsMessage { } // Empty response for the StreamAccessLogs API. Will never be sent. See below. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. message StreamAccessLogsResponse {} // Service for streaming access logs from Envoy to an access log server. @@ -391,7 +415,6 @@ message AccessLog { // The name of the access log implementation to instantiate. The name must // match a statically registered access log. Current built-in loggers include: // 1) "envoy.file_access_log" - // 2) "envoy.grpc_access_log" string name = 1; // Filter which is used to determine if the access log needs to be written. @@ -399,7 +422,6 @@ message AccessLog { // Custom configuration that depends on the access log being instantiated. built-in configurations // include: - // 1) "envoy.file_access_log": FileAccessLog - // 2) "envoy.grpc_access_log": AccessLogServiceConfig + // 1) "envoy.file_access_log": :ref:`FileAccessLog ` google.protobuf.Struct config = 3; } diff --git a/docs/root/api-v1/access_log.rst b/docs/root/api-v1/access_log.rst new file mode 100644 index 00000000..ebe16391 --- /dev/null +++ b/docs/root/api-v1/access_log.rst @@ -0,0 +1,181 @@ +.. _config_access_log_v1: + +Access logging +============== + +Configuration +------------- + +.. code-block:: json + + { + "access_log": [ + { + "path": "...", + "format": "...", + "filter": "{...}", + }, + ] + } + +.. _config_access_log_path_param: + +path + *(required, string)* Path the access log is written to. + +.. _config_access_log_format_param: + +format + *(optional, string)* Access log format. Envoy supports :ref:`custom access log formats + ` as well as a :ref:`default format + `. + +.. _config_access_log_filter_param: + +filter + *(optional, object)* :ref:`Filter ` which is used + to determine if the access log needs to be written. + +.. _config_http_con_manager_access_log_filters_v1: + +Filters +------- + +Envoy supports the following access log filters: + +.. contents:: + :local: + +Status code +^^^^^^^^^^^ + +.. code-block:: json + + { + "filter": { + "type": "status_code", + "op": "...", + "value": "...", + "runtime_key": "..." + } + } + +Filters on HTTP response/status code. + +op + *(required, string)* Comparison operator. Currently *>=* and *=* are the only supported operators. + +value + *(required, integer)* Default value to compare against if runtime value is not available. + +runtime_key + *(optional, string)* Runtime key to get value for comparision. This value is used if defined. + +Duration +^^^^^^^^ + +.. code-block:: json + + { + "filter": { + "type": "duration", + "op": "..", + "value": "...", + "runtime_key": "..." + } + } + +Filters on total request duration in milliseconds. + +op + *(required, string)* Comparison operator. Currently *>=* and *=* are the only supported operators. + +value + *(required, integer)* Default value to compare against if runtime values is not available. + +runtime_key + *(optional, string)* Runtime key to get value for comparision. This value is used if defined. + + +Not health check +^^^^^^^^^^^^^^^^ + +.. code-block:: json + + { + "filter": { + "type": "not_healthcheck" + } + } + +Filters for requests that are not health check requests. A health check request is marked by +the :ref:`health check filter `. + +Traceable +^^^^^^^^^ + +.. code-block:: json + + { + "filter": { + "type": "traceable_request" + } + } + +Filters for requests that are traceable. See the :ref:`tracing overview ` for +more information on how a request becomes traceable. + + +.. _config_http_con_manager_access_log_filters_runtime_v1: + +Runtime +^^^^^^^^^ +.. code-block:: json + + { + "filter": { + "type": "runtime", + "key" : "..." + } + } + +Filters for random sampling of requests. Sampling pivots on the header +:ref:`x-request-id` being present. If +:ref:`x-request-id` is present, the filter will +consistently sample across multiple hosts based on the runtime key value and the value extracted +from :ref:`x-request-id`. If it is missing, the +filter will randomly sample based on the runtime key value. + +key + *(required, string)* Runtime key to get the percentage of requests to be sampled. + This runtime control is specified in the range 0-100 and defaults to 0. + +And +^^^ + +.. code-block:: json + + { + "filter": { + "type": "logical_and", + "filters": [] + } + } + +Performs a logical "and" operation on the result of each filter in *filters*. Filters are evaluated +sequentially and if one of them returns false, the filter returns false immediately. + +Or +^^ + +.. code-block:: json + + { + "filter": { + "type": "logical_or", + "filters": [] + } + } + +Performs a logical "or" operation on the result of each individual filter. Filters are evaluated +sequentially and if one of them returns true, the filter returns true immediately. diff --git a/docs/root/api-v1/api.rst b/docs/root/api-v1/api.rst index 83cf479e..51547326 100644 --- a/docs/root/api-v1/api.rst +++ b/docs/root/api-v1/api.rst @@ -10,6 +10,7 @@ v1 API reference route_config/route_config http_filters/http_filters cluster_manager/cluster_manager + access_log admin rate_limit runtime diff --git a/docs/root/configuration/access_log.rst b/docs/root/configuration/access_log.rst index 6a47047b..a71cae30 100644 --- a/docs/root/configuration/access_log.rst +++ b/docs/root/configuration/access_log.rst @@ -9,35 +9,8 @@ Configuration Access logs are configured as part of the :ref:`HTTP connection manager config ` or :ref:`TCP Proxy `. -.. code-block:: json - - { - "access_log": [ - { - "path": "...", - "format": "...", - "filter": "{...}", - }, - ] - } - -.. _config_access_log_path_param: - -path - *(required, string)* Path the access log is written to. - -.. _config_access_log_format_param: - -format - *(optional, string)* Access log format. Envoy supports :ref:`custom access log formats - ` as well as a :ref:`default format - `. - -.. _config_access_log_filter_param: - -filter - *(optional, object)* :ref:`Filter ` which is used to - determine if the access log needs to be written. +* :ref:`v1 API reference ` +* :ref:`v2 API reference ` .. _config_access_log_format: @@ -147,7 +120,6 @@ The following command operators are supported: TCP Not implemented ("-"). - .. _config_access_log_default_format: Default format @@ -168,147 +140,3 @@ Example of the default Envoy access log format: [2016-04-15T20:17:00.310Z] "POST /api/v1/locations HTTP/2" 204 - 154 0 226 100 "10.0.35.28" "nsq2http" "cc21d9b0-cf5c-432b-8c7e-98aeb7988cd2" "locations" "tcp://10.0.2.1:80" - -.. _config_http_con_manager_access_log_filters: - -Filters -------- - -Envoy supports the following access log filters: - -.. contents:: - :local: - -Status code -^^^^^^^^^^^ - -.. code-block:: json - - { - "filter": { - "type": "status_code", - "op": "...", - "value": "...", - "runtime_key": "..." - } - } - -Filters on HTTP response/status code. - -op - *(required, string)* Comparison operator. Currently *>=* and *=* are the only supported operators. - -value - *(required, integer)* Default value to compare against if runtime value is not available. - -runtime_key - *(optional, string)* Runtime key to get value for comparision. This value is used if defined. - -Duration -^^^^^^^^ - -.. code-block:: json - - { - "filter": { - "type": "duration", - "op": "..", - "value": "...", - "runtime_key": "..." - } - } - -Filters on total request duration in milliseconds. - -op - *(required, string)* Comparison operator. Currently *>=* and *=* are the only supported operators. - -value - *(required, integer)* Default value to compare against if runtime values is not available. - -runtime_key - *(optional, string)* Runtime key to get value for comparision. This value is used if defined. - - -Not health check -^^^^^^^^^^^^^^^^ - -.. code-block:: json - - { - "filter": { - "type": "not_healthcheck" - } - } - -Filters for requests that are not health check requests. A health check request is marked by -the :ref:`health check filter `. - -Traceable -^^^^^^^^^ - -.. code-block:: json - - { - "filter": { - "type": "traceable_request" - } - } - -Filters for requests that are traceable. See the :ref:`tracing overview ` for -more information on how a request becomes traceable. - - -.. _config_http_con_manager_access_log_filters_runtime: - -Runtime -^^^^^^^^^ -.. code-block:: json - - { - "filter": { - "type": "runtime", - "key" : "..." - } - } - -Filters for random sampling of requests. Sampling pivots on the header -:ref:`x-request-id` being present. If -:ref:`x-request-id` is present, the filter will -consistently sample across multiple hosts based on the runtime key value and the value extracted -from :ref:`x-request-id`. If it is missing, the -filter will randomly sample based on the runtime key value. - -key - *(required, string)* Runtime key to get the percentage of requests to be sampled. - This runtime control is specified in the range 0-100 and defaults to 0. - -And -^^^ - -.. code-block:: json - - { - "filter": { - "type": "logical_and", - "filters": [] - } - } - -Performs a logical "and" operation on the result of each filter in *filters*. Filters are evaluated -sequentially and if one of them returns false, the filter returns false immediately. - -Or -^^ - -.. code-block:: json - - { - "filter": { - "type": "logical_or", - "filters": [] - } - } - -Performs a logical "or" operation on the result of each individual filter. Filters are evaluated -sequentially and if one of them returns true, the filter returns true immediately. diff --git a/docs/root/configuration/http_conn_man/headers.rst b/docs/root/configuration/http_conn_man/headers.rst index ca9d48c7..36479a09 100644 --- a/docs/root/configuration/http_conn_man/headers.rst +++ b/docs/root/configuration/http_conn_man/headers.rst @@ -204,7 +204,8 @@ is out of scope for this documentation. If *x-request-id* is propagated across a following features are available: * Stable :ref:`access logging ` via the - :ref:`runtime filter`. + :ref:`v1 API runtime filter` or the + :ref:`v2 API runtime filter`. * Stable tracing when performing random sampling via the :ref:`tracing.random_sampling ` runtime setting or via forced tracing using the :ref:`config_http_conn_man_headers_x-envoy-force-trace` and diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 3b29085c..049b2fb7 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -136,7 +136,7 @@ Version history * :ref:`Auto host rewrite ` during HTTP routing. * :ref:`Regex header matching ` during HTTP routing. -* HTTP access log :ref:`runtime filter `. +* HTTP access log :ref:`runtime filter `. * LightStep tracer :ref:`parent/child span association `. * :ref:`Route discovery service (RDS) API `. * HTTP router :ref:`x-envoy-upstream-rq-timeout-alt-response header