From 1df7f3ad164b95e7fe7bedd32d04101f03784813 Mon Sep 17 00:00:00 2001 From: "data-plane-api(CircleCI)" Date: Thu, 31 Jan 2019 15:04:13 +0000 Subject: [PATCH] tap transport: refactor to use common tap framework (#5724) This commit refactors the tap transport socket to use the common tap extension configuration and tap matching infrastructure. More match conditions will be added in a future PR as well as additional cleanups that have been marked with TODOs. One result of this PR is that the HTTP tap filter can now have a static configuration as well as write to a file per tap sink. All future tap PRs should be smaller and more targeted after this one. Signed-off-by: Matt Klein Mirrored from https://github.com/envoyproxy/envoy @ f37ebdc14f4c0adf0e90aabddae833355c0cec1b --- docs/BUILD | 1 + envoy/config/common/tap/v2alpha/BUILD | 12 +++++++ envoy/config/common/tap/v2alpha/common.proto | 32 +++++++++++++++++ envoy/config/filter/http/tap/v2alpha/BUILD | 2 +- .../config/filter/http/tap/v2alpha/tap.proto | 19 +++------- .../config/transport_socket/tap/v2alpha/BUILD | 1 + .../transport_socket/tap/v2alpha/tap.proto | 35 ++++--------------- envoy/data/tap/v2alpha/BUILD | 5 ++- envoy/data/tap/v2alpha/transport.proto | 8 ++--- envoy/data/tap/v2alpha/wrapper.proto | 4 +++ envoy/service/tap/v2alpha/common.proto | 20 +++++++++++ tools/BUILD | 2 +- tools/data/tap2pcap_h2_ipv4.pb_text | 2 ++ tools/tap2pcap.py | 9 ++--- 14 files changed, 97 insertions(+), 55 deletions(-) create mode 100644 envoy/config/common/tap/v2alpha/BUILD create mode 100644 envoy/config/common/tap/v2alpha/common.proto diff --git a/docs/BUILD b/docs/BUILD index 5a2ab76c..6445a968 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -33,6 +33,7 @@ proto_library( "//envoy/config/accesslog/v2:als", "//envoy/config/accesslog/v2:file", "//envoy/config/bootstrap/v2:bootstrap", + "//envoy/config/common/tap/v2alpha:common", "//envoy/config/filter/accesslog/v2:accesslog", "//envoy/config/filter/http/buffer/v2:buffer", "//envoy/config/filter/http/ext_authz/v2:ext_authz", diff --git a/envoy/config/common/tap/v2alpha/BUILD b/envoy/config/common/tap/v2alpha/BUILD new file mode 100644 index 00000000..dec87953 --- /dev/null +++ b/envoy/config/common/tap/v2alpha/BUILD @@ -0,0 +1,12 @@ +load("//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "common", + srcs = ["common.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/service/tap/v2alpha:common", + ], +) diff --git a/envoy/config/common/tap/v2alpha/common.proto b/envoy/config/common/tap/v2alpha/common.proto new file mode 100644 index 00000000..9fb38328 --- /dev/null +++ b/envoy/config/common/tap/v2alpha/common.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +import "envoy/service/tap/v2alpha/common.proto"; + +import "validate/validate.proto"; + +package envoy.config.common.tap.v2alpha; +option java_package = "io.envoyproxy.envoy.service.tap.v2alpha"; + +// [#protodoc-title: Common tap extension configuration] + +// Common configuration for all tap extensions. +message CommonExtensionConfig { + oneof config_type { + option (validate.required) = true; + + // If specified, the tap filter will be configured via an admin handler. + AdminConfig admin_config = 1; + + // If specified, the tap filter will be configured via a static configuration that cannot be + // changed. + service.tap.v2alpha.TapConfig static_config = 2; + } +} + +// Configuration for the admin handler. See :ref:`here ` for +// more information. +message AdminConfig { + // Opaque configuration ID. When requests are made to the admin handler, the passed opaque ID is + // matched to the configured filter opaque ID to determine which filter to configure. + string config_id = 1 [(validate.rules).string.min_bytes = 1]; +} diff --git a/envoy/config/filter/http/tap/v2alpha/BUILD b/envoy/config/filter/http/tap/v2alpha/BUILD index acec4094..7a826408 100644 --- a/envoy/config/filter/http/tap/v2alpha/BUILD +++ b/envoy/config/filter/http/tap/v2alpha/BUILD @@ -6,6 +6,6 @@ api_proto_library_internal( name = "tap", srcs = ["tap.proto"], deps = [ - "//envoy/service/tap/v2alpha:common", + "//envoy/config/common/tap/v2alpha:common", ], ) diff --git a/envoy/config/filter/http/tap/v2alpha/tap.proto b/envoy/config/filter/http/tap/v2alpha/tap.proto index 8018d5f0..9d93d294 100644 --- a/envoy/config/filter/http/tap/v2alpha/tap.proto +++ b/envoy/config/filter/http/tap/v2alpha/tap.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -import "envoy/service/tap/v2alpha/common.proto"; +import "envoy/config/common/tap/v2alpha/common.proto"; import "validate/validate.proto"; @@ -12,18 +12,7 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.tap.v2alpha"; // Top level configuration for the tap filter. message Tap { - oneof config_type { - option (validate.required) = true; - - // If specified, the tap filter will be configured via an admin handler. - AdminConfig admin_config = 1; - } -} - -// Configuration for the admin handler. See :ref:`here ` for -// more information. -message AdminConfig { - // Opaque configuration ID. When requests are made to the admin handler, the passed opaque ID is - // matched to the configured filter opaque ID to determine which filter to configure. - string config_id = 1 [(validate.rules).string.min_bytes = 1]; + // Common configuration for the HTTP tap filter. + common.tap.v2alpha.CommonExtensionConfig common_config = 1 + [(validate.rules).message.required = true]; } diff --git a/envoy/config/transport_socket/tap/v2alpha/BUILD b/envoy/config/transport_socket/tap/v2alpha/BUILD index a972cb18..e6dcebe3 100644 --- a/envoy/config/transport_socket/tap/v2alpha/BUILD +++ b/envoy/config/transport_socket/tap/v2alpha/BUILD @@ -7,5 +7,6 @@ api_proto_library_internal( srcs = ["tap.proto"], deps = [ "//envoy/api/v2/core:base", + "//envoy/config/common/tap/v2alpha:common", ], ) diff --git a/envoy/config/transport_socket/tap/v2alpha/tap.proto b/envoy/config/transport_socket/tap/v2alpha/tap.proto index ab1e4698..52338f7c 100644 --- a/envoy/config/transport_socket/tap/v2alpha/tap.proto +++ b/envoy/config/transport_socket/tap/v2alpha/tap.proto @@ -6,41 +6,18 @@ option go_package = "v2"; // [#protodoc-title: Tap] +import "envoy/config/common/tap/v2alpha/common.proto"; import "envoy/api/v2/core/base.proto"; -// File sink. -// -// .. warning:: -// -// The current file sink implementation buffers the entire trace in memory -// prior to writing. This will OOM for long lived sockets and/or where there -// is a large amount of traffic on the socket. -message FileSink { - // Path prefix. The output file will be of the form _.pb, where is an - // identifier distinguishing the recorded trace for individual socket instances (the Envoy - // connection ID). - string path_prefix = 1; - - // File format. - enum Format { - // Binary proto format as per :ref:`Trace - // `. - PROTO_BINARY = 0; - // Text proto format as per :ref:`Trace - // `. - PROTO_TEXT = 1; - } - Format format = 2; -} +import "validate/validate.proto"; // Configuration for tap transport socket. This wraps another transport socket, providing the // ability to interpose and record in plain text any traffic that is surfaced to Envoy. message Tap { - oneof sink_selector { - // Trace is to be written to a file sink. - FileSink file_sink = 1; - } + // Common configuration for the tap transport socket. + common.tap.v2alpha.CommonExtensionConfig common_config = 1 + [(validate.rules).message.required = true]; // The underlying transport socket being wrapped. - api.v2.core.TransportSocket transport_socket = 2; + api.v2.core.TransportSocket transport_socket = 2 [(validate.rules).message.required = true]; } diff --git a/envoy/data/tap/v2alpha/BUILD b/envoy/data/tap/v2alpha/BUILD index cdc216c2..2aecd2db 100644 --- a/envoy/data/tap/v2alpha/BUILD +++ b/envoy/data/tap/v2alpha/BUILD @@ -17,5 +17,8 @@ api_proto_library_internal( api_proto_library_internal( name = "wrapper", srcs = ["wrapper.proto"], - deps = [":http"], + deps = [ + ":http", + ":transport", + ], ) diff --git a/envoy/data/tap/v2alpha/transport.proto b/envoy/data/tap/v2alpha/transport.proto index e78a2b9c..216cf9db 100644 --- a/envoy/data/tap/v2alpha/transport.proto +++ b/envoy/data/tap/v2alpha/transport.proto @@ -23,8 +23,8 @@ message Connection { envoy.api.v2.core.Address remote_address = 3; } -// Event in a trace. -message Event { +// Event in a socket trace. +message SocketEvent { // Timestamp for event. google.protobuf.Timestamp timestamp = 1; // Data read by Envoy from the transport socket. @@ -50,9 +50,9 @@ message Event { // Sequence of read/write events that constitute a trace on a socket. // Multiple Trace messages might be emitted for a given connection ID, with the // sink (e.g. file set, network) responsible for later reassembly. -message Trace { +message SocketTrace { // Connection properties. Connection connection = 1; // Sequence of observed events. - repeated Event events = 2; + repeated SocketEvent events = 2; } diff --git a/envoy/data/tap/v2alpha/wrapper.proto b/envoy/data/tap/v2alpha/wrapper.proto index 12a527e8..48a6f863 100644 --- a/envoy/data/tap/v2alpha/wrapper.proto +++ b/envoy/data/tap/v2alpha/wrapper.proto @@ -1,6 +1,7 @@ syntax = "proto3"; import "envoy/data/tap/v2alpha/http.proto"; +import "envoy/data/tap/v2alpha/transport.proto"; import "validate/validate.proto"; @@ -17,5 +18,8 @@ message BufferedTraceWrapper { // An HTTP buffered tap trace. HttpBufferedTrace http_buffered_trace = 1; + + // A buffered socket tap trace. + SocketTrace socket_buffered_trace = 2; } } diff --git a/envoy/service/tap/v2alpha/common.proto b/envoy/service/tap/v2alpha/common.proto index a70bacf5..e4bd6d4a 100644 --- a/envoy/service/tap/v2alpha/common.proto +++ b/envoy/service/tap/v2alpha/common.proto @@ -91,9 +91,29 @@ message OutputSink { // been configured to receive tap configuration from some other source (e.g., static // file, XDS, etc.) configuring the streaming admin output type will fail. StreamingAdminSink streaming_admin = 1; + + // Tap output will be written to a file per tap sink. + FilePerTapSink file_per_tap = 2; } } // Streaming admin sink configuration. message StreamingAdminSink { } + +// The file per tap sink outputs a discrete file for every tapped stream. +message FilePerTapSink { + // Path prefix. The output file will be of the form _.pb, where is an + // identifier distinguishing the recorded trace for stream instances (the Envoy + // connection ID, HTTP stream ID, etc.). + string path_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // File format. + enum Format { + // Binary proto format. + PROTO_BINARY = 0; + // Text proto format. + PROTO_TEXT = 1; + } + Format format = 2 [(validate.rules).enum.defined_only = true]; +} diff --git a/tools/BUILD b/tools/BUILD index 239934f5..b2848aee 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -6,7 +6,7 @@ py_binary( licenses = ["notice"], # Apache 2 visibility = ["//visibility:public"], deps = [ - "//envoy/data/tap/v2alpha:transport_py", + "//envoy/data/tap/v2alpha:wrapper_py", ], ) diff --git a/tools/data/tap2pcap_h2_ipv4.pb_text b/tools/data/tap2pcap_h2_ipv4.pb_text index f5e24499..459b12c2 100644 --- a/tools/data/tap2pcap_h2_ipv4.pb_text +++ b/tools/data/tap2pcap_h2_ipv4.pb_text @@ -1,3 +1,4 @@ +socket_buffered_trace { connection { local_address { socket_address { @@ -66,3 +67,4 @@ events { data: "\000\014\010\000\000\000\000\000\001\"ds\">Advanced searchLanguage tools

© 2018 - Privacy - Terms

\000\000\000\000\001\000\000\000\001" } } +} diff --git a/tools/tap2pcap.py b/tools/tap2pcap.py index 0f61f727..6db37805 100644 --- a/tools/tap2pcap.py +++ b/tools/tap2pcap.py @@ -28,7 +28,7 @@ import time from google.protobuf import text_format -from envoy.data.tap.v2alpha import transport_pb2 +from envoy.data.tap.v2alpha import wrapper_pb2 def DumpEvent(direction, timestamp, data): @@ -44,14 +44,15 @@ def DumpEvent(direction, timestamp, data): def Tap2Pcap(tap_path, pcap_path): - trace = transport_pb2.Trace() + wrapper = wrapper_pb2.BufferedTraceWrapper() if tap_path.endswith('.pb_text'): with open(tap_path, 'r') as f: - text_format.Merge(f.read(), trace) + text_format.Merge(f.read(), wrapper) else: with open(tap_path, 'r') as f: - trace.ParseFromString(f.read()) + wrapper.ParseFromString(f.read()) + trace = wrapper.socket_buffered_trace local_address = trace.connection.local_address.socket_address.address local_port = trace.connection.local_address.socket_address.port_value remote_address = trace.connection.remote_address.socket_address.address