mirror of https://github.com/grpc/grpc.git
Merge pull request #17854 from grpc/revert-17793-py-bm
Revert "Add basic benchmark test for Python"pull/17549/head
Lidi Zheng
6 years ago
committed by
GitHub
commit
588486120d
7 changed files with 20 additions and 444 deletions
@ -1,103 +0,0 @@ |
||||
# Copyright 2019 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
package(default_visibility = ["//visibility:public"]) |
||||
|
||||
load("@grpc_python_dependencies//:requirements.bzl", "requirement") |
||||
|
||||
py_library( |
||||
name = "benchmark_client", |
||||
srcs = ["benchmark_client.py"], |
||||
imports = ["../../"], |
||||
deps = [ |
||||
requirement("six"), |
||||
"//src/proto/grpc/testing:py_benchmark_service_proto", |
||||
"//src/proto/grpc/testing:py_messages_proto", |
||||
"//src/python/grpcio/grpc:grpcio", |
||||
"//src/python/grpcio_tests/tests/unit:resources", |
||||
"//src/python/grpcio_tests/tests/unit:test_common", |
||||
], |
||||
) |
||||
|
||||
py_library( |
||||
name = "benchmark_server", |
||||
srcs = ["benchmark_server.py"], |
||||
imports = ["../../"], |
||||
deps = [ |
||||
"//src/proto/grpc/testing:py_benchmark_service_proto", |
||||
"//src/proto/grpc/testing:py_messages_proto", |
||||
], |
||||
) |
||||
|
||||
py_library( |
||||
name = "client_runner", |
||||
srcs = ["client_runner.py"], |
||||
imports = ["../../"], |
||||
) |
||||
|
||||
py_library( |
||||
name = "histogram", |
||||
srcs = ["histogram.py"], |
||||
imports = ["../../"], |
||||
deps = [ |
||||
"//src/proto/grpc/testing:py_stats_proto", |
||||
], |
||||
) |
||||
|
||||
py_library( |
||||
name = "worker_server", |
||||
srcs = ["worker_server.py"], |
||||
imports = ["../../"], |
||||
deps = [ |
||||
":benchmark_client", |
||||
":benchmark_server", |
||||
":client_runner", |
||||
":histogram", |
||||
"//src/proto/grpc/testing:py_benchmark_service_proto", |
||||
"//src/proto/grpc/testing:py_control_proto", |
||||
"//src/proto/grpc/testing:py_stats_proto", |
||||
"//src/proto/grpc/testing:py_worker_service_proto", |
||||
"//src/python/grpcio/grpc:grpcio", |
||||
"//src/python/grpcio_tests/tests/unit:resources", |
||||
"//src/python/grpcio_tests/tests/unit:test_common", |
||||
], |
||||
) |
||||
|
||||
py_binary( |
||||
name = "qps_worker", |
||||
srcs = ["qps_worker.py"], |
||||
imports = ["../../"], |
||||
main = "qps_worker.py", |
||||
deps = [ |
||||
":worker_server", |
||||
"//src/proto/grpc/testing:py_worker_service_proto", |
||||
"//src/python/grpcio/grpc:grpcio", |
||||
"//src/python/grpcio_tests/tests/unit:test_common", |
||||
], |
||||
) |
||||
|
||||
filegroup( |
||||
name = "scenarios", |
||||
srcs = ["scenarios.json"], |
||||
) |
||||
|
||||
sh_test( |
||||
name = "basic_benchmark_test", |
||||
srcs = ["basic_benchmark_test.sh"], |
||||
data = [ |
||||
":qps_worker", |
||||
":scenarios", |
||||
"//test/cpp/qps:qps_json_driver", |
||||
], |
||||
) |
||||
@ -1,102 +0,0 @@ |
||||
# Python Benchmark Tools |
||||
|
||||
## Scenarios |
||||
|
||||
In `src/proto/grpc/testing/control.proto`, it defines the fields of a scenario. |
||||
In `tools/run_tests/performance/scenario_config.py`, the script generates actual scenario content that usually in json format, or piped to another script. |
||||
|
||||
All Python related benchmark scenarios are: |
||||
* netperf |
||||
* python_generic_sync_streaming_ping_pong |
||||
* python_protobuf_sync_streaming_ping_pong |
||||
* python_protobuf_async_unary_ping_pong |
||||
* python_protobuf_sync_unary_ping_pong |
||||
* python_protobuf_sync_unary_qps_unconstrained |
||||
* python_protobuf_sync_streaming_qps_unconstrained |
||||
* python_protobuf_sync_unary_ping_pong_1MB |
||||
|
||||
Here we picked a small but representative subset, and reduce their benchmark duration from 30 seconds to 10 seconds: |
||||
* python_protobuf_async_unary_ping_pong |
||||
* python_protobuf_sync_streaming_ping_pong |
||||
|
||||
## Why keep the scenario file if it can be generated? |
||||
|
||||
Well... The `tools/run_tests/performance/scenario_config.py` is 1274 lines long. The intention of building these benchmark tools is reducing the complexity of existing infrastructure code. So, instead of calling layers of abstraction to generate the scenario file, keeping a valid static copy is preferable. |
||||
|
||||
Also, if the use case for this tool grows beyond simple static scenarios, we can incorporate automatic generation and selection of scenarios into the tool. |
||||
|
||||
## How to run it? |
||||
|
||||
```shell |
||||
bazel test --test_output=streamed src/python/grpcio_tests/tests/qps:basic_benchmark_test |
||||
``` |
||||
|
||||
## What does the output look like? |
||||
|
||||
``` |
||||
RUNNING SCENARIO: python_protobuf_async_unary_ping_pong |
||||
I0123 00:26:04.746195000 140736237159296 driver.cc:288] Starting server on localhost:10086 (worker #0) |
||||
D0123 00:26:04.747190000 140736237159296 ev_posix.cc:170] Using polling engine: poll |
||||
D0123 00:26:04.747264000 140736237159296 dns_resolver_ares.cc:488] Using ares dns resolver |
||||
I0123 00:26:04.748445000 140736237159296 subchannel.cc:869] Connect failed: {"created":"@1548203164.748403000","description":"Failed to connect to remote host: Connection refused","errno":61,"file":"src/core/lib/iomgr/tcp_client_posix.cc","file_line":207,"os_error":"Connection refused","syscall":"connect","target_address":"ipv6:[::1]:10086"} |
||||
I0123 00:26:04.748585000 140736237159296 subchannel.cc:869] Connect failed: {"created":"@1548203164.748564000","description":"Failed to connect to remote host: Connection refused","errno":61,"file":"src/core/lib/iomgr/tcp_client_posix.cc","file_line":207,"os_error":"Connection refused","syscall":"connect","target_address":"ipv4:127.0.0.1:10086"} |
||||
I0123 00:26:04.748596000 140736237159296 subchannel.cc:751] Subchannel 0x7fca43c19360: Retry in 999 milliseconds |
||||
I0123 00:26:05.751251000 123145571299328 subchannel.cc:710] Failed to connect to channel, retrying |
||||
I0123 00:26:05.752209000 140736237159296 subchannel.cc:832] New connected subchannel at 0x7fca45000060 for subchannel 0x7fca43c19360 |
||||
I0123 00:26:05.772291000 140736237159296 driver.cc:349] Starting client on localhost:10087 (worker #1) |
||||
D0123 00:26:05.772384000 140736237159296 driver.cc:373] Client 0 gets 1 channels |
||||
I0123 00:26:05.773286000 140736237159296 subchannel.cc:832] New connected subchannel at 0x7fca45004a80 for subchannel 0x7fca451034b0 |
||||
I0123 00:26:05.789797000 140736237159296 driver.cc:394] Initiating |
||||
I0123 00:26:05.790858000 140736237159296 driver.cc:415] Warming up |
||||
I0123 00:26:07.791078000 140736237159296 driver.cc:421] Starting |
||||
I0123 00:26:07.791860000 140736237159296 driver.cc:448] Running |
||||
I0123 00:26:17.790915000 140736237159296 driver.cc:462] Finishing clients |
||||
I0123 00:26:17.791821000 140736237159296 driver.cc:476] Received final status from client 0 |
||||
I0123 00:26:17.792148000 140736237159296 driver.cc:508] Finishing servers |
||||
I0123 00:26:17.792493000 140736237159296 driver.cc:522] Received final status from server 0 |
||||
I0123 00:26:17.795786000 140736237159296 report.cc:82] QPS: 2066.6 |
||||
I0123 00:26:17.795799000 140736237159296 report.cc:122] QPS: 2066.6 (258.3/server core) |
||||
I0123 00:26:17.795805000 140736237159296 report.cc:127] Latencies (50/90/95/99/99.9%-ile): 467.9/504.8/539.0/653.3/890.4 us |
||||
I0123 00:26:17.795811000 140736237159296 report.cc:137] Server system time: 100.00% |
||||
I0123 00:26:17.795815000 140736237159296 report.cc:139] Server user time: 100.00% |
||||
I0123 00:26:17.795818000 140736237159296 report.cc:141] Client system time: 100.00% |
||||
I0123 00:26:17.795821000 140736237159296 report.cc:143] Client user time: 100.00% |
||||
I0123 00:26:17.795825000 140736237159296 report.cc:148] Server CPU usage: 0.00% |
||||
I0123 00:26:17.795828000 140736237159296 report.cc:153] Client Polls per Request: 0.00 |
||||
I0123 00:26:17.795831000 140736237159296 report.cc:155] Server Polls per Request: 0.00 |
||||
I0123 00:26:17.795834000 140736237159296 report.cc:160] Server Queries/CPU-sec: 1033.19 |
||||
I0123 00:26:17.795837000 140736237159296 report.cc:162] Client Queries/CPU-sec: 1033.32 |
||||
RUNNING SCENARIO: python_protobuf_sync_streaming_ping_pong |
||||
I0123 00:26:17.795888000 140736237159296 driver.cc:288] Starting server on localhost:10086 (worker #0) |
||||
D0123 00:26:17.795964000 140736237159296 ev_posix.cc:170] Using polling engine: poll |
||||
D0123 00:26:17.795978000 140736237159296 dns_resolver_ares.cc:488] Using ares dns resolver |
||||
I0123 00:26:17.796613000 140736237159296 subchannel.cc:832] New connected subchannel at 0x7fca43c15820 for subchannel 0x7fca43d12140 |
||||
I0123 00:26:17.810911000 140736237159296 driver.cc:349] Starting client on localhost:10087 (worker #1) |
||||
D0123 00:26:17.811037000 140736237159296 driver.cc:373] Client 0 gets 1 channels |
||||
I0123 00:26:17.811892000 140736237159296 subchannel.cc:832] New connected subchannel at 0x7fca43d18f40 for subchannel 0x7fca43d16b80 |
||||
I0123 00:26:17.818902000 140736237159296 driver.cc:394] Initiating |
||||
I0123 00:26:17.820776000 140736237159296 driver.cc:415] Warming up |
||||
I0123 00:26:19.824685000 140736237159296 driver.cc:421] Starting |
||||
I0123 00:26:19.825970000 140736237159296 driver.cc:448] Running |
||||
I0123 00:26:29.821866000 140736237159296 driver.cc:462] Finishing clients |
||||
I0123 00:26:29.823259000 140736237159296 driver.cc:476] Received final status from client 0 |
||||
I0123 00:26:29.827195000 140736237159296 driver.cc:508] Finishing servers |
||||
I0123 00:26:29.827599000 140736237159296 driver.cc:522] Received final status from server 0 |
||||
I0123 00:26:29.828739000 140736237159296 report.cc:82] QPS: 619.5 |
||||
I0123 00:26:29.828752000 140736237159296 report.cc:122] QPS: 619.5 (77.4/server core) |
||||
I0123 00:26:29.828760000 140736237159296 report.cc:127] Latencies (50/90/95/99/99.9%-ile): 1589.8/1854.3/1920.4/2015.8/2204.8 us |
||||
I0123 00:26:29.828765000 140736237159296 report.cc:137] Server system time: 100.00% |
||||
I0123 00:26:29.828769000 140736237159296 report.cc:139] Server user time: 100.00% |
||||
I0123 00:26:29.828772000 140736237159296 report.cc:141] Client system time: 100.00% |
||||
I0123 00:26:29.828776000 140736237159296 report.cc:143] Client user time: 100.00% |
||||
I0123 00:26:29.828780000 140736237159296 report.cc:148] Server CPU usage: 0.00% |
||||
I0123 00:26:29.828784000 140736237159296 report.cc:153] Client Polls per Request: 0.00 |
||||
I0123 00:26:29.828788000 140736237159296 report.cc:155] Server Polls per Request: 0.00 |
||||
I0123 00:26:29.828792000 140736237159296 report.cc:160] Server Queries/CPU-sec: 309.58 |
||||
I0123 00:26:29.828795000 140736237159296 report.cc:162] Client Queries/CPU-sec: 309.75 |
||||
``` |
||||
|
||||
## Future Works (TODOs) |
||||
|
||||
1. Generate a target for each scenario. |
||||
2. Simplify the main entrance of our benchmark related code, or make it depends on Bazel. |
||||
@ -1,45 +0,0 @@ |
||||
#! /bin/bash |
||||
# Copyright 2019 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# This test benchmarks Python client/server. |
||||
set -ex |
||||
|
||||
declare -a DRIVER_PORTS=("10086" "10087") |
||||
SCENARIOS_FILE=src/python/grpcio_tests/tests/qps/scenarios.json |
||||
|
||||
function join { local IFS="$1"; shift; echo "$*"; } |
||||
|
||||
if [[ -e "${SCENARIOS_FILE}" ]]; then |
||||
echo "Running against ${SCENARIOS_FILE}:" |
||||
cat "${SCENARIOS_FILE}" |
||||
else |
||||
echo "Failed to find ${SCENARIOS_FILE}!" |
||||
exit 1 |
||||
fi |
||||
|
||||
echo "Starting Python qps workers..." |
||||
qps_workers=() |
||||
for DRIVER_PORT in "${DRIVER_PORTS[@]}" |
||||
do |
||||
echo -e "\tRunning Python qps worker listening at localhost:${DRIVER_PORT}..." |
||||
src/python/grpcio_tests/tests/qps/qps_worker \ |
||||
--driver_port="${DRIVER_PORT}" & |
||||
qps_workers+=("localhost:${DRIVER_PORT}") |
||||
done |
||||
|
||||
echo "Running qps json driver..." |
||||
QPS_WORKERS=$(join , ${qps_workers[@]}) |
||||
export QPS_WORKERS |
||||
test/cpp/qps/qps_json_driver --scenarios_file="${SCENARIOS_FILE}" |
||||
@ -1,96 +0,0 @@ |
||||
{ |
||||
"scenarios": [ |
||||
{ |
||||
"name": "python_protobuf_async_unary_ping_pong", |
||||
"clientConfig": { |
||||
"clientType": "ASYNC_CLIENT", |
||||
"securityParams": { |
||||
"useTestCa": true, |
||||
"serverHostOverride": "foo.test.google.fr" |
||||
}, |
||||
"outstandingRpcsPerChannel": 1, |
||||
"clientChannels": 1, |
||||
"asyncClientThreads": 1, |
||||
"loadParams": { |
||||
"closedLoop": {} |
||||
}, |
||||
"payloadConfig": { |
||||
"simpleParams": {} |
||||
}, |
||||
"histogramParams": { |
||||
"resolution": 0.01, |
||||
"maxPossible": 60000000000 |
||||
}, |
||||
"channelArgs": [ |
||||
{ |
||||
"name": "grpc.optimization_target", |
||||
"strValue": "latency" |
||||
} |
||||
] |
||||
}, |
||||
"numClients": 1, |
||||
"serverConfig": { |
||||
"serverType": "ASYNC_SERVER", |
||||
"securityParams": { |
||||
"useTestCa": true, |
||||
"serverHostOverride": "foo.test.google.fr" |
||||
}, |
||||
"channelArgs": [ |
||||
{ |
||||
"name": "grpc.optimization_target", |
||||
"strValue": "latency" |
||||
} |
||||
] |
||||
}, |
||||
"numServers": 1, |
||||
"warmupSeconds": 2, |
||||
"benchmarkSeconds": 10 |
||||
}, |
||||
{ |
||||
"name": "python_protobuf_sync_streaming_ping_pong", |
||||
"clientConfig": { |
||||
"securityParams": { |
||||
"useTestCa": true, |
||||
"serverHostOverride": "foo.test.google.fr" |
||||
}, |
||||
"outstandingRpcsPerChannel": 1, |
||||
"clientChannels": 1, |
||||
"asyncClientThreads": 1, |
||||
"rpcType": "STREAMING", |
||||
"loadParams": { |
||||
"closedLoop": {} |
||||
}, |
||||
"payloadConfig": { |
||||
"simpleParams": {} |
||||
}, |
||||
"histogramParams": { |
||||
"resolution": 0.01, |
||||
"maxPossible": 60000000000 |
||||
}, |
||||
"channelArgs": [ |
||||
{ |
||||
"name": "grpc.optimization_target", |
||||
"strValue": "latency" |
||||
} |
||||
] |
||||
}, |
||||
"numClients": 1, |
||||
"serverConfig": { |
||||
"serverType": "ASYNC_SERVER", |
||||
"securityParams": { |
||||
"useTestCa": true, |
||||
"serverHostOverride": "foo.test.google.fr" |
||||
}, |
||||
"channelArgs": [ |
||||
{ |
||||
"name": "grpc.optimization_target", |
||||
"strValue": "latency" |
||||
} |
||||
] |
||||
}, |
||||
"numServers": 1, |
||||
"warmupSeconds": 2, |
||||
"benchmarkSeconds": 10 |
||||
} |
||||
] |
||||
} |
||||
Loading…
Reference in new issue