Adding missing filters (http fault, redis, mongo, tcp, rate limits, etc..) (#192)

Signed-off-by: Shriram Rajagopalan <shriram@us.ibm.com>

.circleci/config.yml

@@ -0,0 +1,25 @@
+version: 2
+jobs:
+   test:
+     docker:
+       - image: lyft/envoy-build:114e24c6fd05fc026492e9d2ca5608694e5ea59d
+     resource_class: xlarge
+     working_directory: /source
+     steps:
+       - checkout
+       - run: ci/do_ci.sh bazel.test
+   docs:
+     docker:
+       - image: lyft/envoy-build:114e24c6fd05fc026492e9d2ca5608694e5ea59d
+     resource_class: xlarge
+     working_directory: /source
+     steps:
+       - checkout
+       - run: ci/do_ci.sh bazel.docs
+
+workflows:
+  version: 2
+  all:
+    jobs:
+      - test
+      - docs

api/address.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package envoy.api.v2;
 
+import "google/protobuf/wrappers.proto";
+
 // [V2-API-DIFF] Addresses now have .proto structure.
 
 message Pipe {
@@ -46,3 +48,10 @@ message Address {
     Pipe pipe = 2;
   }
 }
+
+// CidrRange specifies an IP Address and a prefix length to construct
+// the subnet mask.
+message CidrRange {
+  string address_prefix = 1;
+  google.protobuf.UInt32Value prefix_len = 2;
+}

api/filter/BUILD

@@ -3,26 +3,6 @@ load("//bazel:api_build_system.bzl", "api_proto_library")
 licenses(["notice"])  # Apache 2
 
 api_proto_library(
-    name = "http_connection_manager",
+    name = "fault",
-    srcs = ["http_connection_manager.proto"],
+    srcs = ["fault.proto"],
-    deps = [
-        "//api:base",
-        "//api:protocol",
-        "//api:rds",
-    ],
-)
-
-api_proto_library(
-    name = "mongo_proxy",
-    srcs = ["mongo_proxy.proto"],
-)
-
-api_proto_library(
-    name = "rate_limit",
-    srcs = ["rate_limit.proto"],
-)
-
-api_proto_library(
-    name = "tcp_proxy",
-    srcs = ["tcp_proxy.proto"],
 )

api/filter/README.md

@@ -1,8 +1,9 @@
 ## NOTE
 
-The list of filters here is incomplete. There are no proto specifications for Fault filter, Redis filter, CORS filter, etc.
+If a filter configuration is not captured in the proto specification, you
-These specifications will be added in the near future. In the interim, you can still supply plain JSON configuration objects
+can still supply plain JSON configuration objects for such filters by
-for these missing filters by setting the `"deprecated_v1"` field to true in the filter's configuration. For example,
+setting the `"deprecated_v1"` field to true in the filter's
+configuration. For example,
 
 ```json
 { 

api/filter/fault.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter;
+
+import "google/protobuf/duration.proto";
+
+// Delay specification is used to inject latency into the
+// HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections.
+message FaultDelay {
+  enum FaultDelayType {
+    // Fixed delay (step function).
+    FIXED = 0;
+  }
+
+  // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is supported.
+  FaultDelayType type = 1;
+
+  // An integer between 0-100 indicating the percentage of operations/connection requests
+  // on which the delay will be injected.
+  uint32 percent = 2;
+
+  oneof fault_delay_type {
+    // Add a fixed delay before forwarding the operation upstream. See
+    // https://developers.google.com/protocol-buffers/docs/proto3#json for
+    // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified
+    // delay will be injected before a new request/operation. For TCP
+    // connections, the proxying of the connection upstream will be delayed
+    // for the specified period. REQUIRED.
+    google.protobuf.Duration fixed_delay = 3;
+  }
+}

api/filter/http/BUILD

@@ -0,0 +1,53 @@
+load("//bazel:api_build_system.bzl", "api_proto_library")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library(
+    name = "http_connection_manager",
+    srcs = ["http_connection_manager.proto"],
+    deps = [
+        "//api:base",
+        "//api:protocol",
+        "//api:rds",
+    ],
+)
+
+api_proto_library(
+    name = "router",
+    srcs = ["router.proto"],
+)
+
+api_proto_library(
+    name = "buffer",
+    srcs = ["buffer.proto"],
+)
+
+api_proto_library(
+    name = "transcoder",
+    srcs = ["transcoder.proto"],
+)
+
+api_proto_library(
+    name = "rate_limit",
+    srcs = ["rate_limit.proto"],
+)
+
+api_proto_library(
+    name = "ip_tagging",
+    srcs = ["ip_tagging.proto"],
+    deps = ["//api:address"],
+)
+
+api_proto_library(
+    name = "health_check",
+    srcs = ["health_check.proto"],
+)
+
+api_proto_library(
+    name = "fault",
+    srcs = ["fault.proto"],
+    deps = [
+        "//api:rds",
+        "//api/filter:fault",
+    ],
+)

api/filter/http/buffer.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/wrappers.proto";
+
+// The buffer filter is used to stop filter iteration and wait for a fully
+// buffered complete request. This is useful in different situations
+// including protecting some applications from having to deal with partial
+// requests and high network latency.
+message Buffer {
+  // The maximum request size that the filter will before the connection
+  // manager will stop buffering and return a 413 response. REQUIRED.
+  google.protobuf.UInt32Value max_request_bytes = 1;
+  // The maximum amount of time that the filter will wait for a complete
+  // request before returning a 408 response. REQUIRED.
+  google.protobuf.Duration max_request_time = 2;
+}

api/filter/http/fault.proto

@@ -0,0 +1,77 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+import "api/rds.proto";
+import "api/filter/fault.proto";
+
+// Abort specification is used to prematurely abort a HTTP/gRPC/Mongo/Redis
+// operation/TCP connection with a pre-specified error code.
+message FaultAbort {
+  // An integer between 0-100 indicating the percentage of requests/operations/connections
+  // that will be aborted with the error code provided.
+  uint32 percent = 1;
+
+  // Applicable only for HTTP connections.
+  oneof error_type {
+    // HTTP status code to use to abort the HTTP request.
+    uint32 http_status = 2;
+  }
+}
+
+// The fault injection filter can be used to test the resiliency of
+// microservices to different forms of failures. The filter can be used to
+// inject delays and abort requests with user-specified error codes,
+// thereby providing the ability to stage different failure scenarios such
+// as service failures, service overloads, high network latency, network
+// partitions, etc. Faults injection can be limited to a specific set of
+// requests based on the (destination) upstream cluster of a request and/or
+// a set of pre-defined request headers.
+//
+// The scope of failures is restricted to those that are observable by an
+// application communicating over the network. CPU and disk failures on the
+// local host cannot be emulated.
+//
+// Currently, the fault injection filter has the following limitations:
+//
+// * Abort codes are restricted to HTTP status codes only
+// * Delays are restricted to fixed duration.
+//
+// Future versions will include support for restricting faults to specific
+// routes, and delay durations based on distributions.
+//
+// * Note:* The fault injection filter must be inserted before any
+// other filter, including the router filter.
+message HTTPFault {
+  // If specified, the filter will inject delays based on the values in the
+  // object. At least abort or delay must be specified.
+  FaultDelay delay = 1;
+
+  // If specified, the filter will abort requests based on the values in
+  // the object. At least abort or delay must be specified.
+  FaultAbort abort = 2;
+
+  // Specifies the name of the (destination) upstream cluster that the
+  // filter should match on. Fault injection will be restricted to requests
+  // bound to the specific upstream cluster.
+  string upstream_cluster = 3;
+
+  // Specifies a set of headers that the filter should match on. The fault
+  // injection filter can be applied selectively to requests that match a
+  // set of headers specified in the fault filter config. The chances of
+  // actual fault injection further depend on the values of FaultAbort.percent
+  // and FaultDelay.percent parameters. The filter will check the request’s
+  // headers against all the specified headers in the filter config. A
+  // match will happen if all the headers in the config are present in the
+  // request with the same values (or based on presence if the `value` field
+  // is not in the config). TODO: allow runtime configuration on per entry
+  // basis for headers match.
+  repeated HeaderMatcher headers = 4;
+
+  // Faults are injected for the specified list of downstream hosts. If
+  // this setting is not set, faults are injected for all downstream
+  // nodes. Downstream node name is taken from the HTTP
+  // x-envoy-downstream-service-node header and compared against
+  // downstream_nodes list.
+  repeated string downstream_nodes = 5;
+}

api/filter/http/health_check.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/wrappers.proto";
+
+// Note that the filter will automatically fail health checks and set the
+// x-envoy-immediate-health-check-fail header if the /healthcheck/fail
+// admin endpoint has been called. (The /healthcheck/ok admin endpoint
+// reverses this behavior).
+message HealthCheck {
+  // Specifies whether the filter operates in pass through mode or not. REQUIRED.
+  google.protobuf.BoolValue pass_through_mode = 1;
+  // Specifies the incoming HTTP endpoint that should be considered the
+  // health check endpoint. For example /healthcheck.
+  string endpoint = 2;
+  // If operating in pass through mode, the amount of time in milliseconds
+  // that the filter should cache the upstream response. REQUIRED.
+  google.protobuf.Duration cache_time = 3;
+}

api/filter/http/http_connection_manager.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package envoy.api.v2.filter;
+package envoy.api.v2.filter.http;
 
 import "api/base.proto";
 import "api/protocol.proto";

api/filter/http/ip_tagging.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+import "api/address.proto";
+
+// This is an HTTP filter which enables Envoy to tag requests with extra
+// information such as location, cloud source, and any extra data. This is
+// useful to prevent against DDoS.
+message IPTagging {
+  enum RequestType {
+    BOTH = 0;
+    INTERNAL = 1;
+    EXTERNAL = 2;
+  }
+
+  // The type of requests the filter should apply to. The supported types
+  // are internal, external or both. A request is considered internal if
+  // x-envoy-internal is set to true. If x-envoy-internal is not set or
+  // false, a request is considered external. The filter defaults to both,
+  // and it will apply to all request types.
+  RequestType request_type = 1;
+
+  repeated IPTag ip_tags = 2;
+
+  message IPTag {
+    // Specifies the ip tag name to apply.
+    string ip_tag_name = 1;
+
+    // A list of IP address and subnet masks that will be tagged with the
+    // ip_tag_name. Both IPv4 and IPv6 CIDR addresses are allowed here.
+    repeated CidrRange ip_list = 2;
+  }
+}

api/filter/http/rate_limit.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+import "google/protobuf/duration.proto";
+
+// HTTP rate limit filter configuration. The HTTP rate limit filter will
+// call the rate limit service when the request’s route or virtual host has
+// one or more rate limit configurations that match the filter stage
+// setting. The route can optionally include the virtual host rate limit
+// configurations. More than one configuration can apply to a request. Each
+// configuration results in a descriptor being sent to the rate limit
+// service. If the rate limit service is called, and the response for any
+// of the descriptors is over limit, a 429 response is returned.
+message RateLimit {
+  // The rate limit domain to use when calling the rate limit service.
+  string domain = 1;
+  // Specifies the rate limit configurations to be applied with the same
+  // stage number. If not set, the default stage number is 0. NOTE: The
+  // filter supports a range of 0 - 10 inclusively for stage numbers.
+  uint32 stage = 2;
+  // The type of requests the filter should apply to. The supported types
+  // are internal, external or both. A request is considered internal if
+  // x-envoy-internal is set to true. If x-envoy-internal is not set or
+  // false, a request is considered external. The filter defaults to both,
+  // and it will apply to all request types.
+  string request_type = 3;
+  // The timeout in milliseconds for the rate limit service RPC. If not
+  // set, this defaults to 20ms.
+  google.protobuf.Duration timeout = 4;
+}

api/filter/http/router.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+import "google/protobuf/wrappers.proto";
+
+// The router filter implements HTTP forwarding. It will be used in almost
+// all HTTP proxy scenarios that Envoy is deployed for. The filter’s main
+// job is to follow the instructions specified in the configured route
+// table. In addition to forwarding and redirection, the filter also handles
+// retry, statistics, etc.
+message Router {
+  // Whether the router generates dynamic cluster statistics. Defaults to
+  // true. Can be disabled in high performance scenarios.
+  google.protobuf.BoolValue dynamic_stats = 1;
+  // Whether to start a child span for egress routed calls. This can be
+  // useful in scenarios where other filters (auth, ratelimit, etc.) make
+  // outbound calls and have child spans rooted at the same ingress
+  // parent. Defaults to false.
+  bool start_child_span = 2;
+}

api/filter/http/transcoder.proto

@@ -0,0 +1,52 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.http;
+
+// This is a filter which allows a RESTful JSON API client to send requests
+// to Envoy over HTTP and get proxied to a gRPC service. The HTTP mapping
+// for the gRPC service has to be defined by custom options, defined in
+// https://cloud.google.com/service-management/reference/rpc/google.api#http
+message GrpcJsonTranscoder {
+  // The filter config for the filter requires the descriptor file as well
+  // as a list of the gRPC services to be transcoded.
+
+  // Supplies the binary protobuf descriptor set for the gRPC services. The
+  // descriptor set has to include all of the types that are used in the
+  // services. Make sure to use the --include_import option for protoc.
+  string proto_descriptor = 1;
+  // A list of strings that supplies the service names that the transcoder
+  // will translate. If the service name doesn’t exist in proto_descriptor,
+  // Envoy will fail at startup. The proto_descriptor may contain more
+  // services than the service names specified here, but they won’t be
+  // translated.
+  repeated string services = 2;
+
+  // Control options for response json. These options are passed directly
+  // to JsonPrintOptions.
+  message PrintOptions {
+    // Whether to add spaces, line breaks and indentation to make the JSON
+    // output easy to read. Default to false.
+    bool add_whitespace = 1;
+
+    // Whether to always print primitive fields. By default primitive
+    // fields with default values will be omitted in JSON output. For
+    // example, an int32 field set to 0 will be omitted. Set this flag to
+    // true will override the default behavior and print primitive fields
+    // regardless of their values. Default to false.
+    bool always_print_primitive_fields = 2;
+
+    // Whether to always print enums as ints. By default they are rendered
+    // as strings. Default to false.
+    bool always_print_enums_as_ints = 3;
+
+    // Whether to preserve proto field names. By default protobuf will
+    // generate JSON field names use json_name option, or lower camel case,
+    // in that order. Set this flag will preserve original field
+    // names. Default to false.
+    bool preserve_proto_field_names = 4;
+  };
+
+  // Control options for response json. These options are passed directly
+  // to JsonPrintOptions.
+  PrintOptions print_options = 3;
+}

api/filter/network/BUILD

@@ -0,0 +1,31 @@
+load("//bazel:api_build_system.bzl", "api_proto_library")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library(
+    name = "mongo_proxy",
+    srcs = ["mongo_proxy.proto"],
+    deps = ["//api/filter:fault"],
+)
+
+api_proto_library(
+    name = "tcp_proxy",
+    srcs = ["tcp_proxy.proto"],
+)
+
+api_proto_library(
+    name = "redis_proxy",
+    srcs = ["redis_proxy.proto"],
+)
+
+api_proto_library(
+    name = "client_ssl_auth",
+    srcs = ["client_ssl_auth.proto"],
+    deps = ["//api:address"],
+)
+
+api_proto_library(
+    name = "rate_limit",
+    srcs = ["rate_limit.proto"],
+    deps = ["//api:rls"],
+)

api/filter/network/client_ssl_auth.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.network;
+
+import "api/address.proto";
+import "google/protobuf/duration.proto";
+
+// Envoy provides a network filter that performs TLS client authentication
+// via principals fetched from a REST VPN service. This filter matches the
+// presented client certificate hash against the principal list to
+// determine whether the connection should be allowed or not. Optional IP
+// white listing can also be configured. This functionality can be used to
+// build edge proxy VPN support for web infrastructure.
+message ClientSSLAuth {
+  // The cluster manager cluster that runs the authentication service. The
+  // filter will connect to the service every 60s to fetch the list of
+  // principals. The service must support the expected REST API.
+  string auth_api_cluster = 1;
+  // The prefix to use when emitting statistics.
+  string stat_prefix = 2;
+  // Time in milliseconds between principal refreshes from the
+  // authentication service. Default is 60000 (60s). The actual fetch time
+  // will be this value plus a random jittered value between
+  // 0-refresh_delay_ms milliseconds.
+  google.protobuf.Duration refresh_delay = 3;
+  // An optional list of IP address and subnet masks that should be white
+  // listed for access by the filter. If no list is provided, there is no
+  // IP white list.
+  repeated CidrRange ip_white_list = 4;
+}

api/filter/network/mongo_proxy.proto

@@ -1,6 +1,8 @@
 syntax = "proto3";
 
-package envoy.api.v2.filter;
+package envoy.api.v2.filter.network;
+
+import "api/filter/fault.proto";
 
 message MongoProxy {
   // The human readable prefix to use when emitting statistics for the
@@ -11,4 +13,10 @@ message MongoProxy {
   // path is specified no access logs will be written. Note that access log is
   // also gated by runtime.
   string access_log = 2;
+
+  // Inject a fixed delay before proxying a Mongo operation. Delays are
+  // applied to the following MongoDB operations: Query, Insert, GetMore,
+  // and KillCursors. Once an active delay is in progress, all incoming
+  // data up until the timer event fires will be a part of the delay.
+  FaultDelay delay = 3;
 }

api/filter/network/rate_limit.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.network;
+
+import "api/rls.proto";
+import "google/protobuf/duration.proto";
+
+// TCP rate limit filter configuration
+message RateLimit {
+  // The prefix to use when emitting statistics.
+  string stat_prefix = 1;
+  // The rate limit domain to use in the rate limit service request.
+  string domain = 2;
+  // The rate limit descriptor list to use in the rate limit service request.
+  repeated RateLimitDescriptor descriptors = 3;
+  // The timeout in milliseconds for the rate limit service RPC. If not
+  // set, this defaults to 20ms.
+  google.protobuf.Duration timeout = 4;
+}

api/filter/network/redis_proxy.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+package envoy.api.v2.filter.network;
+
+import "google/protobuf/duration.proto";
+
+message RedisProxy {
+  // The human readable prefix to use when emitting statistics for the
+  // Redis proxy filter. See the statistics documentation for more information.
+  string stat_prefix = 1;
+
+  // Indicates the upstream cluster to which the operation should be routed to.
+  string cluster = 2;
+
+  // Redis connection pool settings.
+  message ConnPoolSettings {
+    // Per-operation timeout. See
+    // https://developers.google.com/protocol-buffers/docs/proto3#json for
+    // the JSON/YAML Duration mapping. The timer starts when the first
+    // command of a pipeline is written to the backend connection. Each
+    // response received from Redis resets the timer since it signifies
+    // that the next command is being processed by the backend. The only
+    // exception to this behavior is when a connection to a backend is not
+    // yet established. In that case, the connect timeout on the cluster
+    // will govern the timeout until the connection is ready. REQUIRED.
+    google.protobuf.Duration op_timeout = 1;
+  }
+
+  // Network settings for the connection pool to the upstream cluster.
+  ConnPoolSettings settings = 3;
+}

api/filter/network/tcp_proxy.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package envoy.api.v2.filter;
+package envoy.api.v2.filter.network;
 
 import "google/protobuf/duration.proto";
 

api/filter/rate_limit.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package envoy.api.v2.filter;
-
-message RateLimit {
-  // The human readable prefix to use when emitting statistics for the
-  // rate limit filter. See the statistics documentation for more information.
-  string stat_prefix = 1;
-
-  // The rate limit domain to use in the rate limit service request.
-  string domain = 2;
-
-  // The rate limit descriptor list to use in the rate limit service request.
-  // TODO(htuch): This should be the shared canonical RateLimitDescriptor when
-  // we import the rate limit protos
-  // (https://github.com/lyft/envoy-api/issues/26).
-  message RateLimitDescriptor {
-    message Entry {
-      string key = 1;
-      string value = 2;
-    }
-    repeated Entry entries = 1;
-  }
-  repeated RateLimitDescriptor rate_limit_descriptors = 3;
-}

api/lds.proto

@@ -56,10 +56,6 @@ message FilterChainMatch {
 
   // If non-empty, an IP address and prefix length to match addresses when the
   // listener is bound to 0.0.0.0/:: or when use_original_dst is specified.
-  message CidrRange {
-    string address_prefix = 1;
-    google.protobuf.UInt32Value prefix_len = 2;
-  }
   repeated CidrRange prefix_ranges = 3;
 
   // If non-empty, an IP address and suffix length to match addresses when the

tools/BUILD

@@ -5,7 +5,7 @@ py_binary(
     visibility = ["//visibility:public"],
     deps = [
         "//api:lds_py",
-        "//api/filter:http_connection_manager_py",
+        "//api/filter/http:http_connection_manager_py",
     ],
 )
 

tools/generate_listeners.py

@@ -17,7 +17,7 @@ from google.protobuf import struct_pb2
 from google.protobuf import text_format
 
 from api import lds_pb2
-from api.filter import http_connection_manager_pb2
+from api.filter.http import http_connection_manager_pb2
 
 
 # Convert an arbitrary proto object to its Struct proto representation.