Merge pull request #14610 from apolcyn/python_dig

Remove the C++ test dependency on dig and dnsutils
pull/14611/head
apolcyn 7 years ago committed by GitHub
commit 44dac6050a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      bazel/grpc_build_system.bzl
  2. 16
      templates/test/cpp/naming/resolver_component_tests_defs.include
  3. 13
      test/cpp/naming/BUILD
  4. 4
      test/cpp/naming/generate_resolver_component_tests.bzl
  5. 40
      test/cpp/naming/resolver_component_tests_runner.sh
  6. 25
      test/cpp/naming/resolver_component_tests_runner_invoker.cc
  7. 50
      test/cpp/naming/utils/BUILD
  8. 48
      test/cpp/naming/utils/dns_resolver.py
  9. 0
      test/cpp/naming/utils/dns_server.py
  10. 35
      test/cpp/naming/utils/tcp_connect.py

@ -174,6 +174,7 @@ def grpc_py_binary(name, srcs, data = [], deps = [], external_deps = [], testonl
native.py_binary(
name = name,
srcs = srcs,
testonly = testonly,
data = data,
deps = deps + _get_external_deps(external_deps)
)

@ -21,9 +21,11 @@ set -ex
FLAGS_test_bin_path=$(echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2)
FLAGS_dns_server_bin_path=$(echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2)
FLAGS_records_config_path=$(echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2)
FLAGS_test_dns_server_port=$(echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2)
FLAGS_dns_server_port=$(echo "$4" | grep '\--dns_server_port=' | cut -d "=" -f 2)
FLAGS_dns_resolver_bin_path=$(echo "$5" | grep '\--dns_resolver_bin_path=' | cut -d "=" -f 2)
FLAGS_tcp_connect_bin_path=$(echo "$6" | grep '\--tcp_connect_bin_path=' | cut -d "=" -f 2)
for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do
for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_dns_server_port" "$FLAGS_dns_resolver_bin_path" "$FLAGS_tcp_connect_bin_path"; do
if [[ "$cmd_arg" == "" ]]; then
echo "Missing a CMD arg" && exit 1
fi
@ -34,17 +36,17 @@ if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then
fi
export GRPC_DNS_RESOLVER=ares
"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" > /dev/null 2>&1 &
"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > /dev/null 2>&1 &
DNS_SERVER_PID=$!
echo "Local DNS server started. PID: $DNS_SERVER_PID"
# Health check local DNS server TCP and UDP ports
for ((i=0;i<30;i++));
do
echo "Retry health-check DNS query to local DNS server over tcp and udp"
echo "Retry health-check local DNS server by attempting a DNS query and TCP handshake"
RETRY=0
dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1
dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1
$FLAGS_dns_resolver_bin_path -s 127.0.0.1 -p "$FLAGS_dns_server_port" -n health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. -t 1 | grep '123.123.123.123' || RETRY=1
$FLAGS_tcp_connect_bin_path -s 127.0.0.1 -p "$FLAGS_dns_server_port" -t 1 || RETRY=1
if [[ "$RETRY" == 0 ]]; then
break
fi;
@ -83,7 +85,7 @@ $FLAGS_test_bin_path \\
--expected_lb_policy='${test['expected_lb_policy']}' \\
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
% endfor

@ -34,17 +34,4 @@ grpc_sh_binary(
],
)
grpc_py_binary(
name = "test_dns_server",
srcs = ["test_dns_server.py"],
testonly = True,
data = [
"resolver_test_record_groups.yaml",
],
external_deps = [
"twisted",
"yaml",
]
)
generate_resolver_component_tests()

@ -54,7 +54,9 @@ def generate_resolver_component_tests():
data = [
":resolver_component_tests_runner",
":resolver_component_test%s" % unsecure_build_config_suffix,
":test_dns_server",
"//test/cpp/naming/utils:dns_server",
"//test/cpp/naming/utils:dns_resolver",
"//test/cpp/naming/utils:tcp_connect",
"resolver_test_record_groups.yaml", # include the transitive dependency so that the dns sever py binary can locate this
],
args = [

@ -21,9 +21,11 @@ set -ex
FLAGS_test_bin_path=$(echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2)
FLAGS_dns_server_bin_path=$(echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2)
FLAGS_records_config_path=$(echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2)
FLAGS_test_dns_server_port=$(echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2)
FLAGS_dns_server_port=$(echo "$4" | grep '\--dns_server_port=' | cut -d "=" -f 2)
FLAGS_dns_resolver_bin_path=$(echo "$5" | grep '\--dns_resolver_bin_path=' | cut -d "=" -f 2)
FLAGS_tcp_connect_bin_path=$(echo "$6" | grep '\--tcp_connect_bin_path=' | cut -d "=" -f 2)
for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do
for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_dns_server_port" "$FLAGS_dns_resolver_bin_path" "$FLAGS_tcp_connect_bin_path"; do
if [[ "$cmd_arg" == "" ]]; then
echo "Missing a CMD arg" && exit 1
fi
@ -34,17 +36,17 @@ if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then
fi
export GRPC_DNS_RESOLVER=ares
"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" > /dev/null 2>&1 &
"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > /dev/null 2>&1 &
DNS_SERVER_PID=$!
echo "Local DNS server started. PID: $DNS_SERVER_PID"
# Health check local DNS server TCP and UDP ports
for ((i=0;i<30;i++));
do
echo "Retry health-check DNS query to local DNS server over tcp and udp"
echo "Retry health-check local DNS server by attempting a DNS query and TCP handshake"
RETRY=0
dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1
dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1
$FLAGS_dns_resolver_bin_path -s 127.0.0.1 -p "$FLAGS_dns_server_port" -n health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. -t 1 | grep '123.123.123.123' || RETRY=1
$FLAGS_tcp_connect_bin_path -s 127.0.0.1 -p "$FLAGS_dns_server_port" -t 1 || RETRY=1
if [[ "$RETRY" == 0 ]]; then
break
fi;
@ -77,7 +79,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -85,7 +87,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -93,7 +95,7 @@ $FLAGS_test_bin_path \
--expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -101,7 +103,7 @@ $FLAGS_test_bin_path \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -109,7 +111,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -117,7 +119,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -125,7 +127,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -133,7 +135,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -141,7 +143,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -149,7 +151,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -157,7 +159,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -165,7 +167,7 @@ $FLAGS_test_bin_path \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
$FLAGS_test_bin_path \
@ -173,7 +175,7 @@ $FLAGS_test_bin_path \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='' \
--local_dns_server_address="127.0.0.1:$FLAGS_test_dns_server_port" &
--local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" &
wait "$!" || EXIT_CODE=1
kill -SIGTERM "$DNS_SERVER_PID" || true

@ -102,14 +102,18 @@ namespace testing {
void InvokeResolverComponentTestsRunner(std::string test_runner_bin_path,
std::string test_bin_path,
std::string dns_server_bin_path,
std::string records_config_path) {
int test_dns_server_port = grpc_pick_unused_port_or_die();
std::string records_config_path,
std::string dns_resolver_bin_path,
std::string tcp_connect_bin_path) {
int dns_server_port = grpc_pick_unused_port_or_die();
SubProcess* test_driver = new SubProcess(
{test_runner_bin_path, "--test_bin_path=" + test_bin_path,
SubProcess* test_driver =
new SubProcess({test_runner_bin_path, "--test_bin_path=" + test_bin_path,
"--dns_server_bin_path=" + dns_server_bin_path,
"--records_config_path=" + records_config_path,
"--test_dns_server_port=" + std::to_string(test_dns_server_port)});
"--dns_server_port=" + std::to_string(dns_server_port),
"--dns_resolver_bin_path=" + dns_resolver_bin_path,
"--tcp_connect_bin_path=" + tcp_connect_bin_path});
gpr_mu test_driver_mu;
gpr_mu_init(&test_driver_mu);
gpr_cv test_driver_cv;
@ -170,8 +174,9 @@ int main(int argc, char** argv) {
// sure that we're using bazel's test environment.
grpc::testing::InvokeResolverComponentTestsRunner(
bin_dir + "/resolver_component_tests_runner",
bin_dir + "/" + FLAGS_test_bin_name, bin_dir + "/test_dns_server",
bin_dir + "/resolver_test_record_groups.yaml");
bin_dir + "/" + FLAGS_test_bin_name, bin_dir + "/utils/dns_server",
bin_dir + "/resolver_test_record_groups.yaml",
bin_dir + "/utils/dns_resolver", bin_dir + "/utils/tcp_connect");
} else {
// Get the current binary's directory relative to repo root to invoke the
// correct build config (asan/tsan/dbg, etc.).
@ -180,8 +185,10 @@ int main(int argc, char** argv) {
grpc::testing::InvokeResolverComponentTestsRunner(
"test/cpp/naming/resolver_component_tests_runner.sh",
bin_dir + "/" + FLAGS_test_bin_name,
"test/cpp/naming/test_dns_server.py",
"test/cpp/naming/resolver_test_record_groups.yaml");
"test/cpp/naming/utils/dns_server.py",
"test/cpp/naming/resolver_test_record_groups.yaml",
"test/cpp/naming/utils/dns_resolver.py",
"test/cpp/naming/utils/tcp_connect.py");
}
grpc_shutdown();
return 0;

@ -0,0 +1,50 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(
default_visibility = ["//visibility:public"],
features = [
"-layering_check",
"-parse_headers",
],
)
licenses(["notice"]) # Apache v2
load("//bazel:grpc_build_system.bzl", "grpc_py_binary")
grpc_py_binary(
name = "dns_server",
srcs = ["dns_server.py"],
testonly = True,
external_deps = [
"twisted",
"yaml",
]
)
grpc_py_binary(
name = "dns_resolver",
srcs = ["dns_resolver.py"],
testonly = True,
external_deps = [
"twisted",
]
)
grpc_py_binary(
name = "tcp_connect",
srcs = ["tcp_connect.py"],
testonly = True,
)

@ -0,0 +1,48 @@
#!/usr/bin/env python2.7
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Makes DNS queries for A records to specified servers"""
import argparse
import signal
import twisted.internet.task as task
import twisted.names.client as client
def main():
argp = argparse.ArgumentParser(description='Make DNS queries for A records')
argp.add_argument('-s', '--server_host', default='127.0.0.1', type=str,
help='Host for DNS server to listen on for TCP and UDP.')
argp.add_argument('-p', '--server_port', default=53, type=int,
help='Port that the DNS server is listening on.')
argp.add_argument('-n', '--qname', default=None, type=str,
help=('Name of the record to query for. '))
argp.add_argument('-t', '--timeout', default=1, type=int,
help=('Force process exit after this number of seconds.'))
args = argp.parse_args()
signal.alarm(args.timeout)
def OnResolverResultAvailable(result):
answers, authority, additional = result
for a in answers:
print(a.payload)
def BeginQuery(reactor, qname):
servers = [(args.server_host, args.server_port)]
resolver = client.Resolver(servers=servers)
deferred_result = resolver.lookupAddress(args.qname)
deferred_result.addCallback(OnResolverResultAvailable)
return deferred_result
task.react(BeginQuery, [args.qname])
if __name__ == '__main__':
main()

@ -0,0 +1,35 @@
#!/usr/bin/env python2.7
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Opens a TCP connection to a specified server and then exits."""
import argparse
import signal
import socket
def main():
argp = argparse.ArgumentParser(description='Open a TCP handshake to a server')
argp.add_argument('-s', '--server_host', default=None, type=str,
help='Server host name or IP.')
argp.add_argument('-p', '--server_port', default=0, type=int,
help='Port that the server is listening on.')
argp.add_argument('-t', '--timeout', default=1, type=int,
help='Force process exit after this number of seconds.')
args = argp.parse_args()
signal.alarm(args.timeout)
socket.create_connection([args.server_host, args.server_port])
if __name__ == '__main__':
main()
Loading…
Cancel
Save