logging: add metadata access log filter (#12322)

Adding a filter for access logs that will decide whether to log based on dynamic metadata.

Signed-off-by: davidraskin <draskin@google.com>

Mirrored from https://github.com/envoyproxy/envoy @ d7c7e9a79eed80afa56a0b05cf6adf0516750d14

envoy/config/accesslog/v3/BUILD

@@ -9,6 +9,7 @@ api_proto_package(
         "//envoy/config/core/v3:pkg",
         "//envoy/config/filter/accesslog/v2:pkg",
         "//envoy/config/route/v3:pkg",
+        "//envoy/type/matcher/v3:pkg",
         "//envoy/type/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
     ],

envoy/config/accesslog/v3/accesslog.proto

@@ -4,10 +4,12 @@ package envoy.config.accesslog.v3;
 
 import "envoy/config/core/v3/base.proto";
 import "envoy/config/route/v3/route_components.proto";
+import "envoy/type/matcher/v3/metadata.proto";
 import "envoy/type/v3/percent.proto";
 
 import "google/protobuf/any.proto";
 import "google/protobuf/struct.proto";
+import "google/protobuf/wrappers.proto";
 
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
@@ -39,8 +41,8 @@ message AccessLog {
   // Filter which is used to determine if the access log needs to be written.
   AccessLogFilter filter = 2;
 
-  // Custom configuration that depends on the access log being instantiated. Built-in
-  // configurations include:
+  // Custom configuration that depends on the access log being instantiated.
+  // Built-in configurations include:
   //
   // #. "envoy.access_loggers.file": :ref:`FileAccessLog
   //    <envoy_api_msg_extensions.access_loggers.file.v3.FileAccessLog>`
@@ -53,7 +55,7 @@ message AccessLog {
   }
 }
 
-// [#next-free-field: 12]
+// [#next-free-field: 13]
 message AccessLogFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.accesslog.v2.AccessLogFilter";
@@ -93,6 +95,9 @@ message AccessLogFilter {
 
     // Extension filter.
     ExtensionFilter extension_filter = 11;
+
+    // Metadata Filter
+    MetadataFilter metadata_filter = 12;
   }
 }
 
@@ -156,25 +161,30 @@ message RuntimeFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.accesslog.v2.RuntimeFilter";
 
-  // Runtime key to get an optional overridden numerator for use in the *percent_sampled* field.
-  // If found in runtime, this value will replace the default numerator.
+  // Runtime key to get an optional overridden numerator for use in the
+  // *percent_sampled* field. If found in runtime, this value will replace the
+  // default numerator.
   string runtime_key = 1 [(validate.rules).string = {min_bytes: 1}];
 
-  // The default sampling percentage. If not specified, defaults to 0% with denominator of 100.
+  // The default sampling percentage. If not specified, defaults to 0% with
+  // denominator of 100.
   type.v3.FractionalPercent percent_sampled = 2;
 
   // By default, sampling pivots on the header
-  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` being present. If
-  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` is present, the filter will
-  // consistently sample across multiple hosts based on the runtime key value and the value
-  // extracted from :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`. If it is
-  // missing, or *use_independent_randomness* is set to true, the filter will randomly sample based
-  // on the runtime key value alone. *use_independent_randomness* can be used for logging kill
-  // switches within complex nested :ref:`AndFilter
+  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` being
+  // present. If :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`
+  // is present, the filter will consistently sample across multiple hosts based
+  // on the runtime key value and the value extracted from
+  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`. If it is
+  // missing, or *use_independent_randomness* is set to true, the filter will
+  // randomly sample based on the runtime key value alone.
+  // *use_independent_randomness* can be used for logging kill switches within
+  // complex nested :ref:`AndFilter
   // <envoy_api_msg_config.accesslog.v3.AndFilter>` and :ref:`OrFilter
-  // <envoy_api_msg_config.accesslog.v3.OrFilter>` blocks that are easier to reason about
-  // from a probability perspective (i.e., setting to true will cause the filter to behave like
-  // an independent random variable when composed within logical operator filters).
+  // <envoy_api_msg_config.accesslog.v3.OrFilter>` blocks that are easier to
+  // reason about from a probability perspective (i.e., setting to true will
+  // cause the filter to behave like an independent random variable when
+  // composed within logical operator filters).
   bool use_independent_randomness = 3;
 }
 
@@ -203,21 +213,22 @@ message HeaderFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.accesslog.v2.HeaderFilter";
 
-  // Only requests with a header which matches the specified HeaderMatcher will pass the filter
-  // check.
+  // Only requests with a header which matches the specified HeaderMatcher will
+  // pass the filter check.
   route.v3.HeaderMatcher header = 1 [(validate.rules).message = {required: true}];
 }
 
 // Filters requests that received responses with an Envoy response flag set.
 // A list of the response flags can be found
-// in the access log formatter :ref:`documentation<config_access_log_format_response_flags>`.
+// in the access log formatter
+// :ref:`documentation<config_access_log_format_response_flags>`.
 message ResponseFlagFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.accesslog.v2.ResponseFlagFilter";
 
-  // Only responses with the any of the flags listed in this field will be logged.
-  // This field is optional. If it is not specified, then any response flag will pass
-  // the filter check.
+  // Only responses with the any of the flags listed in this field will be
+  // logged. This field is optional. If it is not specified, then any response
+  // flag will pass the filter check.
   repeated string flags = 1 [(validate.rules).repeated = {
     items {
       string {
@@ -248,8 +259,8 @@ message ResponseFlagFilter {
   }];
 }
 
-// Filters gRPC requests based on their response status. If a gRPC status is not provided, the
-// filter will infer the status from the HTTP status code.
+// Filters gRPC requests based on their response status. If a gRPC status is not
+// provided, the filter will infer the status from the HTTP status code.
 message GrpcStatusFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.accesslog.v2.GrpcStatusFilter";
@@ -277,11 +288,32 @@ message GrpcStatusFilter {
   // Logs only responses that have any one of the gRPC statuses in this field.
   repeated Status statuses = 1 [(validate.rules).repeated = {items {enum {defined_only: true}}}];
 
-  // If included and set to true, the filter will instead block all responses with a gRPC status or
-  // inferred gRPC status enumerated in statuses, and allow all other responses.
+  // If included and set to true, the filter will instead block all responses
+  // with a gRPC status or inferred gRPC status enumerated in statuses, and
+  // allow all other responses.
   bool exclude = 2;
 }
 
+// Filters based on matching dynamic metadata.
+// If the matcher path and key correspond to an existing key in dynamic
+// metadata, the request is logged only if the matcher value is equal to the
+// metadata value. If the matcher path and key *do not* correspond to an
+// existing key in dynamic metadata, the request is logged only if
+// match_if_key_not_found is "true" or unset.
+message MetadataFilter {
+  option (udpa.annotations.versioning).previous_message_type =
+      "envoy.config.filter.accesslog.v2.MetadataFilter";
+
+  // Matcher to check metadata for specified value. For example, to match on the
+  // access_log_hint metadata, set the filter to "envoy.common" and the path to
+  // "access_log_hint", and the value to "true".
+  type.matcher.v3.MetadataMatcher matcher = 1;
+
+  // Default result if the key does not exist in dynamic metadata: if unset or
+  // true, then log; if false, then don't log.
+  google.protobuf.BoolValue match_if_key_not_found = 2;
+}
+
 // Extension filter is statically registered at runtime.
 message ExtensionFilter {
   option (udpa.annotations.versioning).previous_message_type =

envoy/config/accesslog/v4alpha/BUILD

@@ -9,6 +9,7 @@ api_proto_package(
         "//envoy/config/accesslog/v3:pkg",
         "//envoy/config/core/v4alpha:pkg",
         "//envoy/config/route/v4alpha:pkg",
+        "//envoy/type/matcher/v4alpha:pkg",
         "//envoy/type/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
     ],

envoy/config/accesslog/v4alpha/accesslog.proto

@@ -4,10 +4,12 @@ package envoy.config.accesslog.v4alpha;
 
 import "envoy/config/core/v4alpha/base.proto";
 import "envoy/config/route/v4alpha/route_components.proto";
+import "envoy/type/matcher/v4alpha/metadata.proto";
 import "envoy/type/v3/percent.proto";
 
 import "google/protobuf/any.proto";
 import "google/protobuf/struct.proto";
+import "google/protobuf/wrappers.proto";
 
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
@@ -39,8 +41,8 @@ message AccessLog {
   // Filter which is used to determine if the access log needs to be written.
   AccessLogFilter filter = 2;
 
-  // Custom configuration that depends on the access log being instantiated. Built-in
-  // configurations include:
+  // Custom configuration that depends on the access log being instantiated.
+  // Built-in configurations include:
   //
   // #. "envoy.access_loggers.file": :ref:`FileAccessLog
   //    <envoy_api_msg_extensions.access_loggers.file.v4alpha.FileAccessLog>`
@@ -53,7 +55,7 @@ message AccessLog {
   }
 }
 
-// [#next-free-field: 12]
+// [#next-free-field: 13]
 message AccessLogFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.accesslog.v3.AccessLogFilter";
@@ -93,6 +95,9 @@ message AccessLogFilter {
 
     // Extension filter.
     ExtensionFilter extension_filter = 11;
+
+    // Metadata Filter
+    MetadataFilter metadata_filter = 12;
   }
 }
 
@@ -156,25 +161,30 @@ message RuntimeFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.accesslog.v3.RuntimeFilter";
 
-  // Runtime key to get an optional overridden numerator for use in the *percent_sampled* field.
-  // If found in runtime, this value will replace the default numerator.
+  // Runtime key to get an optional overridden numerator for use in the
+  // *percent_sampled* field. If found in runtime, this value will replace the
+  // default numerator.
   string runtime_key = 1 [(validate.rules).string = {min_bytes: 1}];
 
-  // The default sampling percentage. If not specified, defaults to 0% with denominator of 100.
+  // The default sampling percentage. If not specified, defaults to 0% with
+  // denominator of 100.
   type.v3.FractionalPercent percent_sampled = 2;
 
   // By default, sampling pivots on the header
-  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` being present. If
-  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` is present, the filter will
-  // consistently sample across multiple hosts based on the runtime key value and the value
-  // extracted from :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`. If it is
-  // missing, or *use_independent_randomness* is set to true, the filter will randomly sample based
-  // on the runtime key value alone. *use_independent_randomness* can be used for logging kill
-  // switches within complex nested :ref:`AndFilter
+  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` being
+  // present. If :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`
+  // is present, the filter will consistently sample across multiple hosts based
+  // on the runtime key value and the value extracted from
+  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`. If it is
+  // missing, or *use_independent_randomness* is set to true, the filter will
+  // randomly sample based on the runtime key value alone.
+  // *use_independent_randomness* can be used for logging kill switches within
+  // complex nested :ref:`AndFilter
   // <envoy_api_msg_config.accesslog.v4alpha.AndFilter>` and :ref:`OrFilter
-  // <envoy_api_msg_config.accesslog.v4alpha.OrFilter>` blocks that are easier to reason about
-  // from a probability perspective (i.e., setting to true will cause the filter to behave like
-  // an independent random variable when composed within logical operator filters).
+  // <envoy_api_msg_config.accesslog.v4alpha.OrFilter>` blocks that are easier to
+  // reason about from a probability perspective (i.e., setting to true will
+  // cause the filter to behave like an independent random variable when
+  // composed within logical operator filters).
   bool use_independent_randomness = 3;
 }
 
@@ -202,21 +212,22 @@ message HeaderFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.accesslog.v3.HeaderFilter";
 
-  // Only requests with a header which matches the specified HeaderMatcher will pass the filter
-  // check.
+  // Only requests with a header which matches the specified HeaderMatcher will
+  // pass the filter check.
   route.v4alpha.HeaderMatcher header = 1 [(validate.rules).message = {required: true}];
 }
 
 // Filters requests that received responses with an Envoy response flag set.
 // A list of the response flags can be found
-// in the access log formatter :ref:`documentation<config_access_log_format_response_flags>`.
+// in the access log formatter
+// :ref:`documentation<config_access_log_format_response_flags>`.
 message ResponseFlagFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.accesslog.v3.ResponseFlagFilter";
 
-  // Only responses with the any of the flags listed in this field will be logged.
-  // This field is optional. If it is not specified, then any response flag will pass
-  // the filter check.
+  // Only responses with the any of the flags listed in this field will be
+  // logged. This field is optional. If it is not specified, then any response
+  // flag will pass the filter check.
   repeated string flags = 1 [(validate.rules).repeated = {
     items {
       string {
@@ -247,8 +258,8 @@ message ResponseFlagFilter {
   }];
 }
 
-// Filters gRPC requests based on their response status. If a gRPC status is not provided, the
-// filter will infer the status from the HTTP status code.
+// Filters gRPC requests based on their response status. If a gRPC status is not
+// provided, the filter will infer the status from the HTTP status code.
 message GrpcStatusFilter {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.accesslog.v3.GrpcStatusFilter";
@@ -276,11 +287,32 @@ message GrpcStatusFilter {
   // Logs only responses that have any one of the gRPC statuses in this field.
   repeated Status statuses = 1 [(validate.rules).repeated = {items {enum {defined_only: true}}}];
 
-  // If included and set to true, the filter will instead block all responses with a gRPC status or
-  // inferred gRPC status enumerated in statuses, and allow all other responses.
+  // If included and set to true, the filter will instead block all responses
+  // with a gRPC status or inferred gRPC status enumerated in statuses, and
+  // allow all other responses.
   bool exclude = 2;
 }
 
+// Filters based on matching dynamic metadata.
+// If the matcher path and key correspond to an existing key in dynamic
+// metadata, the request is logged only if the matcher value is equal to the
+// metadata value. If the matcher path and key *do not* correspond to an
+// existing key in dynamic metadata, the request is logged only if
+// match_if_key_not_found is "true" or unset.
+message MetadataFilter {
+  option (udpa.annotations.versioning).previous_message_type =
+      "envoy.config.accesslog.v3.MetadataFilter";
+
+  // Matcher to check metadata for specified value. For example, to match on the
+  // access_log_hint metadata, set the filter to "envoy.common" and the path to
+  // "access_log_hint", and the value to "true".
+  type.matcher.v4alpha.MetadataMatcher matcher = 1;
+
+  // Default result if the key does not exist in dynamic metadata: if unset or
+  // true, then log; if false, then don't log.
+  google.protobuf.BoolValue match_if_key_not_found = 2;
+}
+
 // Extension filter is statically registered at runtime.
 message ExtensionFilter {
   option (udpa.annotations.versioning).previous_message_type =