mirror of https://github.com/grpc/grpc.git
[Interop] Tests for SSA and GAMMA (#34387)
This is just an initial scope of tests. Much of this code was written by @ginayeh . I just did the final polish/integration step. There are 3 main tests included: 1. The GAMMA baseline test, including the [actual GAMMA API](https://gateway-api.sigs.k8s.io/geps/gep-1426/) rather than vendor extensions. 2. Kubernetes-based stateful session affinity tests, where the mesh (including SSA configuration) is configured using CRDs 3. GCP-based stateful session affinity tests, where the mesh is configured using the networkservices APIs directly Tests 1 and 2 will run in both prod and GKE staging, i.e. `container.googleapis.com` and `staging-container.sandbox.googleapis.com`. The latter of these will act as an early detection mechanism for regressions in the controller that translates Gateway resources into networkservices resources. Test 3 will run against `staging-networkservices.sandbox.googleapis.com` to act as an early detection mechanism for regressions in the control plane SSA implementation. The scope of the SSA tests is still fairly minimal. Session drain testing is in-progress but not included in this PR, though several elements required for it are (grace period, pre-stop hook, and the ability to kill a single pod in a deployment). --------- Co-authored-by: Jung-Yu (Gina) Yeh <ginayeh@google.com> Co-authored-by: Sergii Tkachenko <sergiitk@google.com>pull/32941/merge
Richard Belleville
1 year ago
committed by
GitHub
parent
e57b32588b
commit
62521a889f
25 changed files with 1195 additions and 131 deletions
@ -0,0 +1,90 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Utilities for stateful session affinity tests. |
||||
|
||||
These utilities must be shared between test environments that configure SSA |
||||
via Kubernetes CRDs and environments that configure SSA directly through the |
||||
networkservices.googleapis.com API. |
||||
""" |
||||
|
||||
import datetime |
||||
import logging |
||||
from typing import Sequence, Tuple |
||||
|
||||
from framework import xds_k8s_testcase |
||||
from framework.helpers import retryers |
||||
|
||||
_XdsKubernetesBaseTestCase = xds_k8s_testcase.XdsKubernetesBaseTestCase |
||||
_XdsTestServer = xds_k8s_testcase.XdsTestServer |
||||
_XdsTestClient = xds_k8s_testcase.XdsTestClient |
||||
|
||||
_SET_COOKIE_MAX_WAIT_SEC = 300 |
||||
|
||||
|
||||
def get_setcookie_headers( |
||||
metadatas_by_peer: dict[str, "MetadataByPeer"] |
||||
) -> dict[str, str]: |
||||
cookies = dict() |
||||
for peer, metadatas in metadatas_by_peer.items(): |
||||
for rpc_metadatas in metadatas.rpc_metadata: |
||||
for metadata in rpc_metadatas.metadata: |
||||
if metadata.key.lower() == "set-cookie": |
||||
cookies[peer] = metadata.value |
||||
return cookies |
||||
|
||||
|
||||
def assert_eventually_retrieve_cookie_and_server( |
||||
test: _XdsKubernetesBaseTestCase, |
||||
test_client: _XdsTestClient, |
||||
servers: Sequence[_XdsTestServer], |
||||
) -> Tuple[str, _XdsTestServer]: |
||||
"""Retrieves the initial cookie and corresponding server. |
||||
|
||||
Given a test client and set of backends for which SSA is enabled, samples |
||||
a single RPC from the test client to the backends, with metadata collection enabled. |
||||
The "set-cookie" header is retrieved and its contents are returned along with the |
||||
server to which it corresponds. |
||||
|
||||
Since SSA config is supplied as a separate resource from the Route resource, |
||||
there will be periods of time where the SSA config may not be applied. This is |
||||
therefore an eventually consistent function. |
||||
""" |
||||
|
||||
def _assert_retrieve_cookie_and_server(): |
||||
lb_stats = test.assertSuccessfulRpcs(test_client, 1) |
||||
cookies = get_setcookie_headers(lb_stats.metadatas_by_peer) |
||||
test.assertLen(cookies, 1) |
||||
hostname = next(iter(cookies.keys())) |
||||
cookie = cookies[hostname] |
||||
|
||||
chosen_server_candidates = tuple( |
||||
srv for srv in servers if srv.hostname == hostname |
||||
) |
||||
test.assertLen(chosen_server_candidates, 1) |
||||
chosen_server = chosen_server_candidates[0] |
||||
return cookie, chosen_server |
||||
|
||||
retryer = retryers.constant_retryer( |
||||
wait_fixed=datetime.timedelta(seconds=10), |
||||
timeout=datetime.timedelta(seconds=_SET_COOKIE_MAX_WAIT_SEC), |
||||
log_level=logging.INFO, |
||||
) |
||||
try: |
||||
return retryer(_assert_retrieve_cookie_and_server) |
||||
except retryers.RetryError as retry_error: |
||||
logging.exception( |
||||
"Rpcs did not go to expected servers before timeout %s", |
||||
_SET_COOKIE_MAX_WAIT_SEC, |
||||
) |
||||
raise retry_error |
||||
@ -0,0 +1,17 @@ |
||||
--- |
||||
kind: GCPBackendPolicy |
||||
apiVersion: networking.gke.io/v1 |
||||
metadata: |
||||
name: ${be_policy_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
targetRef: |
||||
group: "" |
||||
kind: Service |
||||
name: ${service_name} |
||||
default: |
||||
connectionDraining: |
||||
drainingTimeoutSec: 600 |
||||
... |
||||
@ -0,0 +1,10 @@ |
||||
--- |
||||
apiVersion: v1 |
||||
kind: Service |
||||
metadata: |
||||
name: ${service_name} |
||||
namespace: ${namespace_name} |
||||
spec: |
||||
ports: |
||||
- port: 8080 |
||||
targetPort: 8080 |
||||
@ -0,0 +1,23 @@ |
||||
--- |
||||
kind: HTTPRoute |
||||
apiVersion: gateway.networking.k8s.io/v1beta1 |
||||
metadata: |
||||
name: ${route_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
parentRefs: |
||||
- name: ${frontend_service_name} |
||||
namespace: ${namespace_name} |
||||
group: "" |
||||
kind: Service |
||||
rules: |
||||
- matches: |
||||
- path: |
||||
type: Exact |
||||
value: /grpc.testing.TestService/UnaryCall |
||||
backendRefs: |
||||
- name: ${service_name} |
||||
port: 8080 |
||||
... |
||||
@ -0,0 +1,29 @@ |
||||
--- |
||||
kind: HTTPRoute |
||||
apiVersion: gateway.networking.k8s.io/v1beta1 |
||||
metadata: |
||||
name: ${route_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
parentRefs: |
||||
- name: ${frontend_service_name} |
||||
namespace: ${namespace_name} |
||||
group: "" |
||||
kind: Service |
||||
rules: |
||||
- matches: |
||||
- path: |
||||
type: Exact |
||||
value: /grpc.testing.TestService/UnaryCall |
||||
filters: |
||||
- type: ExtensionRef |
||||
extensionRef: |
||||
group: networking.gke.io |
||||
kind: GCPSessionAffinityFilter |
||||
name: ssa-filter |
||||
backendRefs: |
||||
- name: ${service_name} |
||||
port: 8080 |
||||
... |
||||
@ -0,0 +1,10 @@ |
||||
--- |
||||
apiVersion: networking.gke.io/v1 |
||||
kind: GCPSessionAffinityFilter |
||||
metadata: |
||||
name: ${session_affinity_filter_name} |
||||
namespace: ${namespace_name} |
||||
spec: |
||||
statefulGeneratedCookie: |
||||
cookieTtlSeconds: 50 |
||||
... |
||||
@ -0,0 +1,15 @@ |
||||
--- |
||||
apiVersion: networking.gke.io/v1 |
||||
kind: GCPSessionAffinityPolicy |
||||
metadata: |
||||
name: ${session_affinity_policy_name} |
||||
namespace: ${namespace_name} |
||||
spec: |
||||
statefulGeneratedCookie: |
||||
cookieTtlSeconds: 50 |
||||
targetRef: |
||||
name: ${route_name} |
||||
group: gateway.networking.k8s.io |
||||
kind: HTTPRoute |
||||
namespace: ${namespace_name} |
||||
... |
||||
@ -0,0 +1,15 @@ |
||||
--- |
||||
apiVersion: networking.gke.io/v1 |
||||
kind: GCPSessionAffinityPolicy |
||||
metadata: |
||||
name: ${session_affinity_policy_name} |
||||
namespace: ${namespace_name} |
||||
spec: |
||||
statefulGeneratedCookie: |
||||
cookieTtlSeconds: 50 |
||||
targetRef: |
||||
name: ${service_name} |
||||
kind: Service |
||||
namespace: ${namespace_name} |
||||
group: "" |
||||
... |
||||
@ -1,19 +0,0 @@ |
||||
--- |
||||
kind: TDMesh |
||||
apiVersion: net.gke.io/v1alpha1 |
||||
metadata: |
||||
name: ${mesh_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
gatewayClassName: gke-td |
||||
allowedRoutes: |
||||
namespaces: |
||||
from: All |
||||
kinds: |
||||
- group: net.gke.io |
||||
# This is intentionally incorrect and should be set to GRPCRoute. |
||||
# TODO(sergiitk): [GAMMA] Change when the fix is ready. |
||||
kind: TDGRPCRoute |
||||
... |
||||
@ -0,0 +1,116 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
from typing import List |
||||
|
||||
from absl import flags |
||||
from absl.testing import absltest |
||||
|
||||
from framework import xds_k8s_testcase |
||||
from framework import xds_url_map_testcase |
||||
from framework.test_cases import session_affinity_util |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
flags.adopt_module_key_flags(xds_k8s_testcase) |
||||
|
||||
_XdsTestServer = xds_k8s_testcase.XdsTestServer |
||||
_XdsTestClient = xds_k8s_testcase.XdsTestClient |
||||
RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall |
||||
|
||||
_REPLICA_COUNT = 3 |
||||
|
||||
# This is here temporarily to run this test separately from other app_net tests. |
||||
# TODO(sergiitk): Move into app_net_test.py |
||||
|
||||
|
||||
class AppNetSsaTest(xds_k8s_testcase.AppNetXdsKubernetesTestCase): |
||||
def test_session_affinity_policy(self): |
||||
test_servers: List[_XdsTestServer] |
||||
|
||||
with self.subTest("0_create_health_check"): |
||||
self.td.create_health_check() |
||||
|
||||
with self.subTest("1_create_backend_service"): |
||||
self.td.create_backend_service() |
||||
|
||||
with self.subTest("2_create_mesh"): |
||||
self.td.create_mesh() |
||||
|
||||
with self.subTest("3_create_http_route"): |
||||
self.td.create_http_route_with_content( |
||||
{ |
||||
"meshes": [self.td.mesh.url], |
||||
"hostnames": [ |
||||
f"{self.server_xds_host}:{self.server_xds_port}" |
||||
], |
||||
"rules": [ |
||||
{ |
||||
"action": { |
||||
"destinations": [ |
||||
{ |
||||
"serviceName": self.td.netsvc.resource_full_name( |
||||
self.td.backend_service.name, |
||||
"backendServices", |
||||
), |
||||
"weight": 1, |
||||
}, |
||||
], |
||||
"statefulSessionAffinity": { |
||||
"cookieTtl": "50s", |
||||
}, |
||||
}, |
||||
}, |
||||
], |
||||
} |
||||
) |
||||
|
||||
with self.subTest("4_run_test_server"): |
||||
test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) |
||||
|
||||
with self.subTest("5_setup_server_backends"): |
||||
self.setupServerBackends() |
||||
|
||||
# Default is round robin LB policy. |
||||
|
||||
with self.subTest("6_start_test_client"): |
||||
test_client: _XdsTestClient = self.startTestClient( |
||||
test_servers[0], config_mesh=self.td.mesh.name |
||||
) |
||||
|
||||
with self.subTest("7_send_first_RPC_and_retrieve_cookie"): |
||||
( |
||||
cookie, |
||||
chosen_server, |
||||
) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( |
||||
self, test_client, test_servers |
||||
) |
||||
|
||||
with self.subTest("8_send_RPCs_with_cookie"): |
||||
test_client.update_config.configure( |
||||
rpc_types=(RpcTypeUnaryCall,), |
||||
metadata=( |
||||
( |
||||
RpcTypeUnaryCall, |
||||
"cookie", |
||||
cookie, |
||||
), |
||||
), |
||||
) |
||||
self.assertRpcsEventuallyGoToGivenServers( |
||||
test_client, [chosen_server], 10 |
||||
) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
absltest.main(failfast=True) |
||||
@ -0,0 +1,170 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
from typing import List, Optional |
||||
|
||||
from absl import flags |
||||
from absl.testing import absltest |
||||
|
||||
from framework import xds_gamma_testcase |
||||
from framework import xds_k8s_testcase |
||||
from framework import xds_url_map_testcase |
||||
from framework.rpc import grpc_testing |
||||
from framework.test_app import client_app |
||||
from framework.test_app import server_app |
||||
from framework.test_cases import session_affinity_util |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
flags.adopt_module_key_flags(xds_k8s_testcase) |
||||
|
||||
_XdsTestServer = server_app.XdsTestServer |
||||
_XdsTestClient = client_app.XdsTestClient |
||||
RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall |
||||
|
||||
_REPLICA_COUNT = 3 |
||||
|
||||
|
||||
class AffinityTest(xds_gamma_testcase.GammaXdsKubernetesTestCase): |
||||
def getClientRpcStats( |
||||
self, |
||||
test_client: _XdsTestClient, |
||||
num_rpcs: int, |
||||
*, |
||||
metadata_keys: Optional[tuple[str, ...]] = None, |
||||
) -> grpc_testing.LoadBalancerStatsResponse: |
||||
"""Load all metadata_keys by default.""" |
||||
return super().getClientRpcStats( |
||||
test_client, |
||||
num_rpcs, |
||||
metadata_keys=metadata_keys or client_app.REQ_LB_STATS_METADATA_ALL, |
||||
) |
||||
|
||||
def test_session_affinity_filter(self): |
||||
test_servers: List[_XdsTestServer] |
||||
with self.subTest("01_run_test_server"): |
||||
test_servers = self.startTestServers( |
||||
replica_count=_REPLICA_COUNT, |
||||
route_template="gamma/route_http_ssafilter.yaml", |
||||
) |
||||
|
||||
with self.subTest("02_create_ssa_filter"): |
||||
self.server_runner.createSessionAffinityFilter() |
||||
|
||||
# Default is round robin LB policy. |
||||
|
||||
with self.subTest("03_start_test_client"): |
||||
test_client: _XdsTestClient = self.startTestClient(test_servers[0]) |
||||
|
||||
with self.subTest("04_send_first_RPC_and_retrieve_cookie"): |
||||
( |
||||
cookie, |
||||
chosen_server, |
||||
) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( |
||||
self, test_client, test_servers |
||||
) |
||||
|
||||
with self.subTest("05_send_RPCs_with_cookie"): |
||||
test_client.update_config.configure( |
||||
rpc_types=(RpcTypeUnaryCall,), |
||||
metadata=( |
||||
( |
||||
RpcTypeUnaryCall, |
||||
"cookie", |
||||
cookie, |
||||
), |
||||
), |
||||
) |
||||
self.assertRpcsEventuallyGoToGivenServers( |
||||
test_client, [chosen_server], 10 |
||||
) |
||||
|
||||
def test_session_affinity_policy_with_route_target(self): |
||||
test_servers: List[_XdsTestServer] |
||||
with self.subTest("01_run_test_server"): |
||||
test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) |
||||
|
||||
with self.subTest("02_create_ssa_policy"): |
||||
self.server_runner.createSessionAffinityPolicy( |
||||
"gamma/session_affinity_policy_route.yaml" |
||||
) |
||||
|
||||
# Default is round robin LB policy. |
||||
|
||||
with self.subTest("03_start_test_client"): |
||||
test_client: _XdsTestClient = self.startTestClient(test_servers[0]) |
||||
|
||||
with self.subTest("04_send_first_RPC_and_retrieve_cookie"): |
||||
( |
||||
cookie, |
||||
chosen_server, |
||||
) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( |
||||
self, test_client, test_servers |
||||
) |
||||
|
||||
with self.subTest("05_send_RPCs_with_cookie"): |
||||
test_client.update_config.configure( |
||||
rpc_types=(RpcTypeUnaryCall,), |
||||
metadata=( |
||||
( |
||||
RpcTypeUnaryCall, |
||||
"cookie", |
||||
cookie, |
||||
), |
||||
), |
||||
) |
||||
self.assertRpcsEventuallyGoToGivenServers( |
||||
test_client, [chosen_server], 10 |
||||
) |
||||
|
||||
def test_session_affinity_policy_with_service_target(self): |
||||
test_servers: List[_XdsTestServer] |
||||
with self.subTest("01_run_test_server"): |
||||
test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) |
||||
|
||||
with self.subTest("02_create_ssa_policy"): |
||||
self.server_runner.createSessionAffinityPolicy( |
||||
"gamma/session_affinity_policy_service.yaml" |
||||
) |
||||
|
||||
# Default is round robin LB policy. |
||||
|
||||
with self.subTest("03_start_test_client"): |
||||
test_client: _XdsTestClient = self.startTestClient(test_servers[0]) |
||||
|
||||
with self.subTest("04_send_first_RPC_and_retrieve_cookie"): |
||||
( |
||||
cookie, |
||||
chosen_server, |
||||
) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( |
||||
self, test_client, test_servers |
||||
) |
||||
|
||||
with self.subTest("05_send_RPCs_with_cookie"): |
||||
test_client.update_config.configure( |
||||
rpc_types=(RpcTypeUnaryCall,), |
||||
metadata=( |
||||
( |
||||
RpcTypeUnaryCall, |
||||
"cookie", |
||||
cookie, |
||||
), |
||||
), |
||||
) |
||||
self.assertRpcsEventuallyGoToGivenServers( |
||||
test_client, [chosen_server], 10 |
||||
) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
absltest.main(failfast=True) |
||||
Loading…
Reference in new issue