add streaming access log API (#213)

Signed-off-by: Matt Klein <mklein@lyft.com>

api/BUILD

@@ -7,7 +7,6 @@ licenses(["notice"])  # Apache 2
 proto_library(
     name = "go_protos",
     srcs = [
-        "accesslog.proto",
         "address.proto",
         "base.proto",
         "bootstrap.proto",

api/accesslog.proto

@@ -1,171 +0,0 @@
-syntax = "proto3";
-
-package envoy.api.v2;
-
-import "api/base.proto";
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/struct.proto";
-import "google/protobuf/timestamp.proto";
-import "google/protobuf/wrappers.proto";
-
-message EnvoyAccessLog {
-  // The HTTP request method (RFC 7231/2616)
-  RequestMethod request_method = 1;
-
-  // The time that Envoy started servicing this request
-  google.protobuf.Timestamp start_time = 2;
-
-  // Incoming protocol variation spoken
-  enum Protocol {
-    PROTOCOL_UNSPECIFIED = 0;
-    HTTP10 = 1;
-    HTTP11 = 2;
-    HTTP2 = 3;
-  }
-  Protocol protocol_variant = 3;
-
-  // This enum defines the various things that may have occurred while
-  // processing a request.
-  enum ResponseFlag {
-    // Local server healthcheck failed.
-    FAILED_LOCAL_HEALTHCHECK = 0;
-    // No healthy upstream.
-    NO_HEALTHY_UPSTREAM = 1;
-    // Request timeout on upstream.
-    UPSTREAM_REQUEST_TIMEOUT = 2;
-    // Local codec level reset was sent on the stream.
-    LOCAL_RESET = 3;
-    // Remote codec level reset was received on the stream.
-    UPSTREAM_REMOTE_RESET = 4;
-    // Local reset by a connection pool due to an initial connection failure.
-    UPSTREAM_CONNECTION_FAILURE = 5;
-    // If the stream was locally reset due to connection termination.
-    UPSTREAM_CONNECTION_TERMINATION = 6;
-    // The stream was reset because of a resource overflow.
-    UPSTREAM_OVERFLOW = 7;
-    // No route found for a given request.
-    NO_ROUTE_FOUND = 8;
-    // Request was delayed before proxying.
-    DELAY_INJECTED = 9;
-    // Abort with error code was injected.
-    FAULT_INJECTED = 10;
-    // Request was ratelimited locally by rate limit filter.
-    RATE_LIMITED = 11;
-  }
-  // Status flags about the response.
-  repeated ResponseFlag response_flags = 4;
-
-  // The upstream host URL (Envoy connects to).
-  //
-  // For example, tcp://ip:port for TCP connections.
-  //
-  // IPv6 addresses should be stored in canonical (compressed) format using
-  // [address]:port notation.
-  string upstream_host = 5;
-
-  // The Upstream Cluster that the upstream host belongs to.
-  string upstream_cluster = 6;
-
-  // This field is the IP and port on which the request from the user was
-  // received, stored in ipv4:port or [ipv6]:port format.
-  string destination_host = 7;
-
-  // Size of the HTTP request body in bytes
-  google.protobuf.UInt64Value request_body_bytes = 8;
-
-  // Size of the HTTP response body in bytes
-  google.protobuf.UInt64Value response_body_bytes = 9;
-
-  // Size of the HTTP request headers in bytes
-  google.protobuf.UInt64Value request_headers_bytes = 10;
-
-  // Size of the HTTP response headers in bytes
-  google.protobuf.UInt64Value response_headers_bytes = 11;
-
-  // Whether the request arrived via a secure (TLS) protocol
-  google.protobuf.BoolValue secure = 12;
-
-  // Whether the request is a HealthCheck request
-  google.protobuf.BoolValue health_check = 13;
-
-  // The HTTP response code
-  google.protobuf.UInt32Value response_code = 14;
-
-  // User agent as sent by client HTTP
-  string user_agent = 15;
-
-  // Path
-  //
-  // This is the Path portion from the incoming request URI
-  string path = 17;
-
-  // Referer header as sent by client HTTP
-  // (Referer is spelled to match the HTTP spec, not English).
-  string referer = 18;
-
-  // X-Forwarded-For request header
-  string forwarded_for = 19;
-
-  // X-Request-Id request header
-  //
-  // This header is used by Envoy to uniquely identify a request.
-  // It will be generated for all external requests and internal requests that
-  // do not already have a request ID.  So this field can be guaranteed to exist
-  // and be unique for request tracing purposes.
-  string request_id = 20;
-
-  // HTTP2 :authority header value or HTTP1.1 Host header value
-  string authority = 21;
-
-  // Duration (milliseconds)
-  //
-  // The total duration it took to service this request from the StartTime until
-  // the response was written to the user.
-  google.protobuf.Duration response_duration = 22;
-
-  // Upstream Service Time Duration
-  //
-  // From the X-Envoy-Upstream-Service-Time response header. This is the amount it took
-  // the upstream server to service the request.
-  google.protobuf.Duration upstream_service_duration = 23;
-
-
-  // Original Path from the X-Envoy-Original-Path header.
-  string original_path = 24;
-
-  // All metadata encountered during request processing, including endpoint
-  // selection.
-  //
-  // This can be used to associate IDs attached to the various configurations
-  // used to process this request with the access log entry.  For example, a
-  // route created from a higher level forwarding rule with some ID can place
-  // that ID in this field and cross reference later. It can also be used to
-  // determine if a canary endpoint was used or not.
-  google.protobuf.Struct metadata = 25;
-
-  // Headers configured for logging but not covered by a specific field.
-  repeated HeaderValue request_headers = 26;
-  repeated HeaderValue response_headers = 27;
-
-  // SNI hostname from handshake.
-  string tls_sni_hostname = 28;
-
-  // TLS Version or VERSION_UNSPECIFIED if TLS was not used
-  enum TLSVersion {
-    VERSION_UNSPECIFIED = 0;
-    TLSv1 = 1;
-    TLSv1_1 = 2;
-    TLSv1_2 = 3;
-    TLSv1_3 = 4;
-  }
-  TLSVersion tls_version = 29;
-
-  // TLS Cipher suite negotiated during TLS handshake.
-  // The value is four hex digits defined by the IANA TLS Cipher Suite Registry,
-  // eg, "009C" for TLS_RSA_WITH_AES_128_GCM_SHA256.
-  //
-  // Here is is expressed as an integer.
-  google.protobuf.UInt32Value tls_cipher_suite = 30;
-}
-

api/filter/BUILD

@@ -15,6 +15,8 @@ proto_library(
         "//api:go_protos",
         "@com_google_protobuf//:duration_proto",
         "@com_google_protobuf//:struct_proto",
+        "@com_google_protobuf//:timestamp_proto",
+        "@com_google_protobuf//:wrappers_proto",
     ],
 )
 
@@ -27,6 +29,8 @@ go_proto_library(
         "//api:go_default_library",
         "@com_github_golang_protobuf//ptypes/duration:go_default_library",
         "@com_github_golang_protobuf//ptypes/struct:go_default_library",
+        "@com_github_golang_protobuf//ptypes/timestamp:go_default_library",
+        "@com_github_golang_protobuf//ptypes/wrappers:go_default_library",
     ],
 )
 
@@ -38,5 +42,6 @@ api_proto_library(
 api_proto_library(
     name = "accesslog",
     srcs = ["accesslog.proto"],
+    has_services = 1,
     deps = ["//api:base"],
 )

api/filter/accesslog.proto

@@ -4,7 +4,170 @@ package envoy.api.v2.filter;
 
 import "api/base.proto";
 
+import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
+
+message AccessLogEntry {
+  // The HTTP request method (RFC 7231/2616)
+  RequestMethod request_method = 1;
+
+  // The time that Envoy started servicing this request
+  google.protobuf.Timestamp start_time = 2;
+
+  // Incoming protocol variation spoken
+  enum Protocol {
+    PROTOCOL_UNSPECIFIED = 0;
+    HTTP10 = 1;
+    HTTP11 = 2;
+    HTTP2 = 3;
+  }
+  Protocol protocol_variant = 3;
+
+  // This enum defines the various things that may have occurred while
+  // processing a request.
+  enum ResponseFlag {
+    // Local server healthcheck failed.
+    FAILED_LOCAL_HEALTHCHECK = 0;
+    // No healthy upstream.
+    NO_HEALTHY_UPSTREAM = 1;
+    // Request timeout on upstream.
+    UPSTREAM_REQUEST_TIMEOUT = 2;
+    // Local codec level reset was sent on the stream.
+    LOCAL_RESET = 3;
+    // Remote codec level reset was received on the stream.
+    UPSTREAM_REMOTE_RESET = 4;
+    // Local reset by a connection pool due to an initial connection failure.
+    UPSTREAM_CONNECTION_FAILURE = 5;
+    // If the stream was locally reset due to connection termination.
+    UPSTREAM_CONNECTION_TERMINATION = 6;
+    // The stream was reset because of a resource overflow.
+    UPSTREAM_OVERFLOW = 7;
+    // No route found for a given request.
+    NO_ROUTE_FOUND = 8;
+    // Request was delayed before proxying.
+    DELAY_INJECTED = 9;
+    // Abort with error code was injected.
+    FAULT_INJECTED = 10;
+    // Request was ratelimited locally by rate limit filter.
+    RATE_LIMITED = 11;
+  }
+  // Status flags about the response.
+  repeated ResponseFlag response_flags = 4;
+
+  // The upstream host URL (Envoy connects to).
+  //
+  // For example, tcp://ip:port for TCP connections.
+  //
+  // IPv6 addresses should be stored in canonical (compressed) format using
+  // [address]:port notation.
+  string upstream_host = 5;
+
+  // The Upstream Cluster that the upstream host belongs to.
+  string upstream_cluster = 6;
+
+  // This field is the IP and port on which the request from the user was
+  // received, stored in ipv4:port or [ipv6]:port format.
+  string destination_host = 7;
+
+  // Size of the HTTP request body in bytes
+  google.protobuf.UInt64Value request_body_bytes = 8;
+
+  // Size of the HTTP response body in bytes
+  google.protobuf.UInt64Value response_body_bytes = 9;
+
+  // Size of the HTTP request headers in bytes
+  google.protobuf.UInt64Value request_headers_bytes = 10;
+
+  // Size of the HTTP response headers in bytes
+  google.protobuf.UInt64Value response_headers_bytes = 11;
+
+  // Whether the request arrived via a secure (TLS) protocol
+  google.protobuf.BoolValue secure = 12;
+
+  // Whether the request is a HealthCheck request
+  google.protobuf.BoolValue health_check = 13;
+
+  // The HTTP response code
+  google.protobuf.UInt32Value response_code = 14;
+
+  // User agent as sent by client HTTP
+  string user_agent = 15;
+
+  // Path
+  //
+  // This is the Path portion from the incoming request URI
+  string path = 17;
+
+  // Referer header as sent by client HTTP
+  // (Referer is spelled to match the HTTP spec, not English).
+  string referer = 18;
+
+  // X-Forwarded-For request header
+  string forwarded_for = 19;
+
+  // X-Request-Id request header
+  //
+  // This header is used by Envoy to uniquely identify a request.
+  // It will be generated for all external requests and internal requests that
+  // do not already have a request ID.  So this field can be guaranteed to exist
+  // and be unique for request tracing purposes.
+  string request_id = 20;
+
+  // HTTP2 :authority header value or HTTP1.1 Host header value
+  string authority = 21;
+
+  // Duration (milliseconds)
+  //
+  // The total duration it took to service this request from the StartTime until
+  // the response was written to the user.
+  google.protobuf.Duration response_duration = 22;
+
+  // Upstream Service Time Duration
+  //
+  // From the X-Envoy-Upstream-Service-Time response header. This is the amount it took
+  // the upstream server to service the request.
+  google.protobuf.Duration upstream_service_duration = 23;
+
+
+  // Original Path from the X-Envoy-Original-Path header.
+  string original_path = 24;
+
+  // All metadata encountered during request processing, including endpoint
+  // selection.
+  //
+  // This can be used to associate IDs attached to the various configurations
+  // used to process this request with the access log entry.  For example, a
+  // route created from a higher level forwarding rule with some ID can place
+  // that ID in this field and cross reference later. It can also be used to
+  // determine if a canary endpoint was used or not.
+  google.protobuf.Struct metadata = 25;
+
+  // Headers configured for logging but not covered by a specific field.
+  repeated HeaderValue request_headers = 26;
+  repeated HeaderValue response_headers = 27;
+
+  // SNI hostname from handshake.
+  string tls_sni_hostname = 28;
+
+  // TLS Version or VERSION_UNSPECIFIED if TLS was not used
+  enum TLSVersion {
+    VERSION_UNSPECIFIED = 0;
+    TLSv1 = 1;
+    TLSv1_1 = 2;
+    TLSv1_2 = 3;
+    TLSv1_3 = 4;
+  }
+  TLSVersion tls_version = 29;
+
+  // TLS Cipher suite negotiated during TLS handshake.
+  // The value is four hex digits defined by the IANA TLS Cipher Suite Registry,
+  // eg, "009C" for TLS_RSA_WITH_AES_128_GCM_SHA256.
+  //
+  // Here is is expressed as an integer.
+  google.protobuf.UInt32Value tls_cipher_suite = 30;
+}
 
 // Filter on some integer comparison.
 message ComparisonFilter {
@@ -95,16 +258,67 @@ message FileAccessLog {
   string format = 2;
 }
 
+// Configuration for the built-in "envoy.grpc_access_log" access log type.
+message AccessLogServiceConfig {
+  // The friendly name of the access log to be returned in StreamAccessLogsMessage.Identifier. This
+  // allows the access log server to differentiate between different access logs coming from the
+  // same Envoy.
+  string log_name = 1;
+
+  // The name of the upstream cluster that hosts the access log service. The cluster must be
+  // configured in the cluster manager.
+  string cluster_name = 2;
+}
+
+// Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream
+// access logs without ever expecting a response.
+message StreamAccessLogsMessage {
+  message Identifier {
+    // The node sending the access log messages over the stream.
+    Node node = 1;
+
+    // The friendly name of the log configured in AccessLogServiceConfig.
+    string log_name = 2;
+  }
+
+  // Identifier data that will only be sent in the first message on the stream. This is effectively
+  // structured metadata and is a performance optimization.
+  Identifier identifier = 1;
+
+  // A list of access logs.
+  repeated AccessLogEntry logs = 2;
+}
+
+// Empty response for the StreamAccessLogs API. Will never be sent. See below.
+message StreamAccessLogsResponse {}
+
+// Service for streaming access logs from Envoy to an access log server.
+service AccessLogService {
+  // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any
+  // response to be sent as nothing would be done in the case of failure. The server should
+  // disconnect if it expects Envoy to reconnect. In the future we may decide to add a different
+  // API for "critical" access logs in which Envoy will buffer access logs for some period of time
+  // until it gets an ACK so it could then retry. This API is designed for high throughput with the
+  // expectation that it might be lossy.
+  rpc StreamAccessLogs(stream StreamAccessLogsMessage)
+      returns (StreamAccessLogsResponse) {
+  }
+}
+
 message AccessLog {
   // The name of the access log implementation to instantiate. The name must
-  // match a statically registered access log.
+  // match a statically registered access log. Current built-in loggers include:
+  // 1) "envoy.file_access_log"
+  // 2) "envoy.grpc_access_log"
   string name = 1;
 
   // Filter which is used to determine if the access log needs to be written.
   AccessLogFilter filter = 2;
 
-  // Custom configuration that depends on the access log being instantiated.
+  // Custom configuration that depends on the access log being instantiated. built-in configurations
-  // See the supported AccessLogs for further documentation.
+  // include:
+  // 1) "envoy.file_access_log": FileAccessLog
+  // 2) "envoy.grpc_access_log": AccessLogServiceConfig
   google.protobuf.Struct config = 3;
 }
 

bazel/api_build_system.bzl

@@ -32,6 +32,7 @@ def api_proto_library(name, srcs = [], deps = [], has_services = 0):
             "@com_google_protobuf//:descriptor_proto",
             "@com_google_protobuf//:duration_proto",
             "@com_google_protobuf//:struct_proto",
+            "@com_google_protobuf//:timestamp_proto",
             "@com_google_protobuf//:wrappers_proto",
             "@googleapis//:http_api_protos_lib",
         ],

test/build/BUILD

@@ -6,6 +6,7 @@ api_cc_test(
     name = "build_test",
     srcs = ["build_test.cc"],
     proto_deps = [
+        "//api/filter:accesslog",
         "//api:cds",
         "//api:discovery",
         "//api:eds",

test/build/build_test.cc

@@ -6,6 +6,7 @@
 // Basic C++ build/link validation for the v2 xDS APIs.
 int main(int argc, char *argv[]) {
   const auto methods = {
+    "envoy.api.v2.filter.AccessLogService.StreamAccessLogs",
     "envoy.api.v2.AggregatedDiscoveryService.StreamAggregatedResources",
     "envoy.api.v2.ClusterDiscoveryService.FetchClusters",
     "envoy.api.v2.ClusterDiscoveryService.StreamClusters",