tap: implement HTTP streaming tapping (#6043)

Users can now choose between buffered tapping (simpler) and
streaming tapping (more flexible but harder to work with).

Streaming tapping for the transport socket will be added in a
follow-up.

Signed-off-by: Matt Klein <mklein@lyft.com>

Mirrored from https://github.com/envoyproxy/envoy @ e2e4baaa85a98b14f2bee6ea5aa16dd79cb832d4

envoy/api/v2/core/base.proto

@@ -164,6 +164,11 @@ message HeaderValueOption {
   google.protobuf.BoolValue append = 2;
 }
 
+// Wrapper for a set of headers.
+message HeaderMap {
+  repeated HeaderValue headers = 1;
+}
+
 // Data source consisting of either a file or an inline value.
 message DataSource {
   oneof specifier {

envoy/data/tap/v2alpha/http.proto

@@ -31,3 +31,30 @@ message HttpBufferedTrace {
   // Response message.
   Message response = 2;
 }
+
+// A streamed trace segment. Multiple segments make up a full trace.
+message HttpStreamedTraceSegment {
+  // Trace ID unique to the originating Envoy only. Trace IDs can repeat and should not be used
+  // for long term stable uniqueness.
+  uint64 trace_id = 1;
+
+  oneof message_piece {
+    // Request headers.
+    api.v2.core.HeaderMap request_headers = 2;
+
+    // Request body chunk.
+    Body request_body_chunk = 3;
+
+    // Request trailers.
+    api.v2.core.HeaderMap request_trailers = 4;
+
+    // Response headers.
+    api.v2.core.HeaderMap response_headers = 5;
+
+    // Response body chunk.
+    Body response_body_chunk = 6;
+
+    // Response trailers.
+    api.v2.core.HeaderMap response_trailers = 7;
+  }
+}

envoy/data/tap/v2alpha/transport.proto

@@ -58,10 +58,8 @@ message SocketEvent {
   }
 }
 
-// Sequence of read/write events that constitute a trace on a socket.
-// Multiple Trace messages might be emitted for a given connection ID, with the
-// sink (e.g. file set, network) responsible for later reassembly.
-message SocketTrace {
+// Sequence of read/write events that constitute a buffered trace on a socket.
+message SocketBufferedTrace {
   // Connection properties.
   Connection connection = 1;
 

envoy/data/tap/v2alpha/wrapper.proto

@@ -13,16 +13,19 @@ option java_package = "io.envoyproxy.envoy.data.tap.v2alpha";
 
 // [#protodoc-title: Tap data wrappers]
 
-// Wrapper for all fully buffered tap traces that Envoy emits. This is required for sending traces
-// over gRPC APIs or more easily persisting binary messages to files.
-message BufferedTraceWrapper {
+// Wrapper for all fully buffered and streamed tap traces that Envoy emits. This is required for
+// sending traces over gRPC APIs or more easily persisting binary messages to files.
+message TraceWrapper {
   oneof trace {
     option (validate.required) = true;
 
     // An HTTP buffered tap trace.
     HttpBufferedTrace http_buffered_trace = 1;
 
-    // A buffered socket tap trace.
-    SocketTrace socket_buffered_trace = 2;
+    // An HTTP streamed tap trace segment.
+    HttpStreamedTraceSegment http_streamed_trace_segment = 2;
+
+    // A socket buffered tap trace.
+    SocketBufferedTrace socket_buffered_trace = 3;
   }
 }

envoy/service/tap/v2alpha/common.proto

@@ -90,12 +90,20 @@ message OutputConfig {
   // <envoy_api_field_data.tap.v2alpha.Body.truncated>` field will be set. If not specified, the
   // default is 1KiB.
   google.protobuf.UInt32Value max_buffered_tx_bytes = 3;
+
+  // Indicates whether taps produce a single buffered message per tap, or multiple streamed
+  // messages per tap in the emitted :ref:`TraceWrapper
+  // <envoy_api_msg_data.tap.v2alpha.TraceWrapper>` messages. Note that streamed tapping does not
+  // mean that no buffering takes place. Buffering may be required if data is processed before a
+  // match can be determined. See the HTTP tap filter :ref:`streaming
+  // <config_http_filters_tap_streaming>` documentation for more information.
+  bool streaming = 4;
 }
 
 // Tap output sink configuration.
 message OutputSink {
-  // Output format. All output is in the form of one or more :ref:`BufferedTraceWrapper
-  // <envoy_api_msg_data.tap.v2alpha.BufferedTraceWrapper>` messages. This enumeration indicates
+  // Output format. All output is in the form of one or more :ref:`TraceWrapper
+  // <envoy_api_msg_data.tap.v2alpha.TraceWrapper>` messages. This enumeration indicates
   // how those messages are written. Note that not all sinks support all output formats. See
   // individual sink documentation for more information.
   enum Format {
@@ -121,8 +129,15 @@ message OutputSink {
     // this output format makes consumption simpler.
     PROTO_BINARY = 2;
 
+    // Messages are written as a sequence tuples, where each tuple is the message length encoded
+    // as a `protobuf 32-bit varint
+    // <https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.io.coded_stream>`_
+    // followed by the binary message. The messages can be read back using the language specific
+    // protobuf coded stream implementation to obtain the message length and the message.
+    PROTO_BINARY_LENGTH_DELIMITED = 3;
+
     // Text proto format.
-    PROTO_TEXT = 3;
+    PROTO_TEXT = 4;
   }
 
   // Sink output format.

tools/tap2pcap.py

@@ -44,7 +44,7 @@ def DumpEvent(direction, timestamp, data):
 
 
 def Tap2Pcap(tap_path, pcap_path):
-  wrapper = wrapper_pb2.BufferedTraceWrapper()
+  wrapper = wrapper_pb2.TraceWrapper()
   if tap_path.endswith('.pb_text'):
     with open(tap_path, 'r') as f:
       text_format.Merge(f.read(), wrapper)