tap: introduce HTTP tap filter (#5515)

This is a MVP for the HTTP tap filter. It includes minimal
infrastructure for the following:
1. Generic tap configuration which in the future will be used for
   static config, XDS config, etc. In this MVP the tap can be
   configured via a /tap admin endpoint.
2. Generic output configuration which in the future will be used for
   different output sinks such as files, gRPC API, etc. In this MVP
   the tap results are streamed back out the /tap admin endpoint.
3. Matching infrastructure. In this MVP only matching on request and
   response headers are implemented. Both logical AND and logical OR
   matches are possible.
4. In this MVP request/response body is not considered at all.
5. All docs are included and with all the caveats the filter is ready
   to use for the limited cases it supports (which are likely still to
   be useful).

There is a lot of follow on work which I will do in subsequent PRs.
This includes:
1. Merging the existing capture transport socket into this framework.
2. Implementing body support, both for matching on body contents as
   well as outputting body data.
3. Tap rate limiting so too many streams do not get tapped.
4. gRPC matching. Using reflection and loaded proto definitions, it will
   be possible to match on gRPC fields.
5. JSON matching. If the body parses as JSON, we can allow matching on
   JSON fields.

Part of https://github.com/envoyproxy/envoy/issues/1413.

Signed-off-by: Matt Klein <mklein@lyft.com>

Mirrored from https://github.com/envoyproxy/envoy @ cf80045587240d494e54e9772949bc9af5eda61f

docs/BUILD

@@ -18,6 +18,7 @@ proto_library(
         "//envoy/admin/v2alpha:memory",
         "//envoy/admin/v2alpha:mutex_stats",
         "//envoy/admin/v2alpha:server_info",
+        "//envoy/admin/v2alpha:tap",
         "//envoy/api/v2:cds",
         "//envoy/api/v2:discovery",
         "//envoy/api/v2:eds",
@@ -46,6 +47,7 @@ proto_library(
         "//envoy/config/filter/http/rbac/v2:rbac",
         "//envoy/config/filter/http/router/v2:router",
         "//envoy/config/filter/http/squash/v2:squash",
+        "//envoy/config/filter/http/tap/v2alpha:tap",
         "//envoy/config/filter/http/transcoder/v2:transcoder",
         "//envoy/config/filter/listener/original_src/v2alpha1:original_src",
         "//envoy/config/filter/network/client_ssl_auth/v2:client_ssl_auth",
@@ -72,6 +74,8 @@ proto_library(
         "//envoy/data/accesslog/v2:accesslog",
         "//envoy/data/core/v2alpha:health_check_event",
         "//envoy/data/tap/v2alpha:capture",
+        "//envoy/data/tap/v2alpha:http",
+        "//envoy/data/tap/v2alpha:wrapper",
         "//envoy/service/accesslog/v2:als",
         "//envoy/service/auth/v2alpha:attribute_context",
         "//envoy/service/auth/v2alpha:external_auth",
@@ -79,6 +83,7 @@ proto_library(
         "//envoy/service/load_stats/v2:lrs",
         "//envoy/service/metrics/v2:metrics_service",
         "//envoy/service/ratelimit/v2:rls",
+        "//envoy/service/tap/v2alpha:common",
         "//envoy/type:percent",
         "//envoy/type:range",
         "//envoy/type/matcher:metadata",

envoy/admin/v2alpha/BUILD

@@ -55,3 +55,11 @@ api_proto_library_internal(
     srcs = ["server_info.proto"],
     visibility = ["//visibility:public"],
 )
+
+api_proto_library_internal(
+    name = "tap",
+    srcs = ["tap.proto"],
+    deps = [
+        "//envoy/service/tap/v2alpha:common",
+    ],
+)

envoy/admin/v2alpha/tap.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+import "envoy/service/tap/v2alpha/common.proto";
+import "validate/validate.proto";
+
+package envoy.admin.v2alpha;
+option java_package = "io.envoyproxy.envoy.admin.v2alpha";
+
+// The /tap admin request body that is used to configure an active tap session.
+message TapRequest {
+  // The opaque configuration ID used to match the configuration to a loaded extension.
+  // A tap extension configures a similar opaque ID that is used to match.
+  string config_id = 1 [(validate.rules).string.min_bytes = 1];
+
+  // The tap configuration to load.
+  service.tap.v2alpha.TapConfig tap_config = 2 [(validate.rules).message.required = true];
+}

envoy/config/filter/http/tap/v2alpha/BUILD

@@ -0,0 +1,11 @@
+load("//bazel:api_build_system.bzl", "api_proto_library_internal")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library_internal(
+    name = "tap",
+    srcs = ["tap.proto"],
+    deps = [
+        "//envoy/service/tap/v2alpha:common",
+    ],
+)

envoy/config/filter/http/tap/v2alpha/tap.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+import "envoy/service/tap/v2alpha/common.proto";
+
+import "validate/validate.proto";
+
+package envoy.config.filter.http.tap.v2alpha;
+option java_package = "io.envoyproxy.envoy.config.filter.http.tap.v2alpha";
+
+// [#protodoc-title: Tap]
+// Tap :ref:`configuration overview <config_http_filters_tap>`.
+
+// Top level configuration for the tap filter.
+message Tap {
+  oneof config_type {
+    option (validate.required) = true;
+
+    // If specified, the tap filter will be configured via an admin handler.
+    AdminConfig admin_config = 1;
+  }
+}
+
+// Configuration for the admin handler. See :ref:`here <config_http_filters_tap_admin_handler>` for
+// more information.
+message AdminConfig {
+  // Opaque configuration ID. When requests are made to the admin handler, the passed opaque ID is
+  // matched to the configured filter opaque ID to determine which filter to configure.
+  string config_id = 1 [(validate.rules).string.min_bytes = 1];
+}

envoy/data/tap/v2alpha/BUILD

@@ -7,3 +7,15 @@ api_proto_library_internal(
     srcs = ["capture.proto"],
     deps = ["//envoy/api/v2/core:address"],
 )
+
+api_proto_library_internal(
+    name = "http",
+    srcs = ["http.proto"],
+    deps = ["//envoy/api/v2/core:base"],
+)
+
+api_proto_library_internal(
+    name = "wrapper",
+    srcs = ["wrapper.proto"],
+    deps = [":http"],
+)

envoy/data/tap/v2alpha/capture.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-// [#protodoc-title: Common TAP]
+// [#protodoc-title: Common tap]
 // Trace capture format for the capture transport socket extension. This dumps plain text read/write
 // sequences on a socket.
 

envoy/data/tap/v2alpha/http.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package envoy.data.tap.v2alpha;
+option java_package = "io.envoyproxy.envoy.data.tap.v2alpha";
+
+import "envoy/api/v2/core/base.proto";
+
+// [#protodoc-title: HTTP tap data]
+
+// A fully buffered HTTP trace message.
+message HttpBufferedTrace {
+  // Request headers.
+  repeated api.v2.core.HeaderValue request_headers = 2;
+
+  // Response headers.
+  repeated api.v2.core.HeaderValue response_headers = 3;
+}

envoy/data/tap/v2alpha/wrapper.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+import "envoy/data/tap/v2alpha/http.proto";
+
+import "validate/validate.proto";
+
+package envoy.data.tap.v2alpha;
+option java_package = "io.envoyproxy.envoy.data.tap.v2alpha";
+
+// [#protodoc-title: Tap data wrappers]
+
+// Wrapper for all fully buffered tap traces that Envoy emits. This is required for sending traces
+// over gRPC APIs or more easily persisting binary messages to files.
+message BufferedTraceWrapper {
+  oneof trace {
+    option (validate.required) = true;
+
+    // An HTTP buffered tap trace.
+    HttpBufferedTrace http_buffered_trace = 1;
+  }
+}

envoy/service/tap/v2alpha/BUILD

@@ -0,0 +1,12 @@
+load("//bazel:api_build_system.bzl", "api_proto_library_internal")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library_internal(
+    name = "common",
+    srcs = ["common.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//envoy/api/v2/route",
+    ],
+)

envoy/service/tap/v2alpha/common.proto

@@ -0,0 +1,99 @@
+syntax = "proto3";
+
+import "envoy/api/v2/route/route.proto";
+
+import "validate/validate.proto";
+
+package envoy.service.tap.v2alpha;
+option java_package = "io.envoyproxy.envoy.service.tap.v2alpha";
+
+// [#protodoc-title: Common tap configuration]
+
+// Tap configuration.
+message TapConfig {
+  // The match configuration. If the configuration matches the data source being tapped, a tap will
+  // occur, with the result written to the configured output.
+  MatchPredicate match_config = 1 [(validate.rules).message.required = true];
+
+  // The tap output configuration. If a match configuration matches a data source being tapped,
+  // a tap will occur and the data will be written to the configured output.
+  OutputConfig output_config = 2 [(validate.rules).message.required = true];
+
+  // [#comment:TODO(mattklein123): Rate limiting]
+}
+
+// Tap match configuration. This is a recursive structure which allows complex nested match
+// configurations to be built using various logical operators.
+message MatchPredicate {
+  // A set of match configurations used for logical operations.
+  message MatchSet {
+    // The list of rules that make up the set.
+    repeated MatchPredicate rules = 1 [(validate.rules).repeated .min_items = 2];
+  }
+
+  oneof rule {
+    option (validate.required) = true;
+
+    // A set that describes a logical OR. If any member of the set matches, the match configuration
+    // matches.
+    MatchSet or_match = 1;
+
+    // A set that describes a logical AND. If all members of the set match, the match configuration
+    // matches.
+    MatchSet and_match = 2;
+
+    // A negation match. The match configuration will match if the negated match condition matches.
+    MatchPredicate not_match = 3;
+
+    // The match configuration will always match.
+    bool any_match = 4 [(validate.rules).bool.const = true];
+
+    // HTTP request match configuration.
+    HttpRequestMatch http_request_match = 5;
+
+    // HTTP response match configuration.
+    HttpResponseMatch http_response_match = 6;
+  }
+}
+
+// HTTP request match configuration.
+message HttpRequestMatch {
+  // HTTP request headers to match.
+  repeated api.v2.route.HeaderMatcher headers = 1;
+}
+
+// HTTP response match configuration.
+message HttpResponseMatch {
+  // HTTP response headers to match.
+  repeated api.v2.route.HeaderMatcher headers = 1;
+}
+
+// Tap output configuration.
+message OutputConfig {
+  // Output sinks for tap data. Currently a single sink is allowed in the list. Once multiple
+  // sink types are supported this constraint will be relaxed.
+  repeated OutputSink sinks = 1 [(validate.rules).repeated = {min_items: 1, max_items: 1}];
+
+  // [#comment:TODO(mattklein123): Output filtering. E.g., certain headers, truncated body, etc.]
+}
+
+// Tap output sink configuration.
+message OutputSink {
+  oneof output_sink_type {
+    option (validate.required) = true;
+
+    // Tap output will be streamed out the :http:post:`/tap` admin endpoint.
+    //
+    // .. attention::
+    //
+    //   It is only allowed to specify the streaming admin output sink if the tap is being
+    //   configured from the :http:post:`/tap` admin endpoint. Thus, if an extension has
+    //   been configured to receive tap configuration from some other source (e.g., static
+    //   file, XDS, etc.) configuring the streaming admin output type will fail.
+    StreamingAdminSink streaming_admin = 1;
+  }
+}
+
+// Streaming admin sink configuration.
+message StreamingAdminSink {
+}