diff --git a/src/objective-c/tests/run_one_test.sh b/src/objective-c/tests/run_one_test.sh new file mode 100644 index 00000000000..04cba0e321a --- /dev/null +++ b/src/objective-c/tests/run_one_test.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Don't run this script standalone. Instead, run from the repository root: +# ./tools/run_tests/run_tests.py -l objc + +set -ev + +cd $(dirname $0) + +BINDIR=../../../bins/$CONFIG + +[ -f $BINDIR/interop_server ] || { + echo >&2 "Can't find the test server. Make sure run_tests.py is making" \ + "interop_server before calling this script." + exit 1 +} + +[ -z "$(ps aux |egrep 'port_server\.py.*-p\s32766')" ] && { + echo >&2 "Can't find the port server. Start port server with tools/run_tests/start_port_server.py." + exit 1 +} + +PLAIN_PORT=$(curl localhost:32766/get) +TLS_PORT=$(curl localhost:32766/get) + +$BINDIR/interop_server --port=$PLAIN_PORT --max_send_message_size=8388608 & +$BINDIR/interop_server --port=$TLS_PORT --max_send_message_size=8388608 --use_tls & + +trap 'kill -9 `jobs -p` ; echo "EXIT TIME: $(date)"' EXIT + +set -o pipefail + +XCODEBUILD_FILTER='(^CompileC |^Ld |^ *[^ ]*clang |^ *cd |^ *export |^Libtool |^ *[^ ]*libtool |^CpHeader |^ *builtin-copy )' + +xcodebuild \ + -workspace Tests.xcworkspace \ + -scheme SCHEME \ + -destination name="iPhone 8" \ + HOST_PORT_LOCALSSL=localhost:$TLS_PORT \ + HOST_PORT_LOCAL=localhost:$PLAIN_PORT \ + HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ + test \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - diff --git a/tools/internal_ci/macos/grpc_basictests_objc_examples.cfg b/tools/internal_ci/macos/grpc_basictests_objc_examples.cfg new file mode 100644 index 00000000000..bdfd62c7937 --- /dev/null +++ b/tools/internal_ci/macos/grpc_basictests_objc_examples.cfg @@ -0,0 +1,31 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results --extra_args -r ios-buildtest-.*" +} diff --git a/tools/internal_ci/macos/grpc_basictests_objc_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_objc_ios.cfg similarity index 88% rename from tools/internal_ci/macos/grpc_basictests_objc_dbg.cfg rename to tools/internal_ci/macos/grpc_basictests_objc_ios.cfg index 068961234be..faef331cab5 100644 --- a/tools/internal_ci/macos/grpc_basictests_objc_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_objc_ios.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos objc dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results --extra_args -r ios-test-.*" } diff --git a/tools/internal_ci/macos/grpc_basictests_objc_mac.cfg b/tools/internal_ci/macos/grpc_basictests_objc_mac.cfg new file mode 100644 index 00000000000..19849c357f3 --- /dev/null +++ b/tools/internal_ci/macos/grpc_basictests_objc_mac.cfg @@ -0,0 +1,31 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results --extra_args -r mac-test-.*" +} diff --git a/tools/internal_ci/macos/grpc_basictests_objc_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_examples.cfg similarity index 95% rename from tools/internal_ci/macos/grpc_basictests_objc_opt.cfg rename to tools/internal_ci/macos/pull_request/grpc_basictests_objc_examples.cfg index 927fa50deb0..e2e55777ad1 100644 --- a/tools/internal_ci/macos/grpc_basictests_objc_opt.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_examples.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --extra_args -r ios-buildtest-.*" } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_objc_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_ios.cfg similarity index 96% rename from tools/internal_ci/macos/pull_request/grpc_basictests_objc_opt.cfg rename to tools/internal_ci/macos/pull_request/grpc_basictests_objc_ios.cfg index 652ef1bb77a..4a4539b2353 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_objc_opt.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_ios.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --max_time=3600" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --extra_args -r ios-test-.*" } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_objc_dbg.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_mac.cfg similarity index 90% rename from tools/internal_ci/macos/pull_request/grpc_basictests_objc_dbg.cfg rename to tools/internal_ci/macos/pull_request/grpc_basictests_objc_mac.cfg index 775fd355a5b..98cf9345280 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_objc_dbg.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_mac.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos objc dbg --internal_ci -j 1 --inner_jobs 4 --max_time=3600" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --extra_args -r mac-test-.*" } diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 43d8c64da0b..e5506063091 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1057,54 +1057,82 @@ class ObjCLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): - return [ - self.config.job_spec( - ['src/objective-c/tests/run_tests.sh'], - timeout_seconds=60 * 60, - shortname='objc-tests', - cpu_cost=1e6, - environ=_FORCE_ENVIRON_FOR_WRAPPERS), - self.config.job_spec( - ['src/objective-c/tests/run_plugin_tests.sh'], - timeout_seconds=60 * 60, - shortname='objc-plugin-tests', - cpu_cost=1e6, - environ=_FORCE_ENVIRON_FOR_WRAPPERS), - self.config.job_spec( + out = [] + if self.config == 'dbg': + out += self.config.job_spec( ['src/objective-c/tests/build_one_example.sh'], timeout_seconds=10 * 60, - shortname='objc-build-example-sample', + shortname='ios-buildtest-example-sample', cpu_cost=1e6, environ={ 'SCHEME': 'Sample', 'EXAMPLE_PATH': 'src/objective-c/examples/Sample' - }), - self.config.job_spec( + }) + out += job_spec( ['src/objective-c/tests/build_one_example.sh'], timeout_seconds=10 * 60, - shortname='objc-build-example-sample-frameworks', + shortname='ios-buildtest-example-sample-frameworks', cpu_cost=1e6, environ={ 'SCHEME': 'Sample', 'EXAMPLE_PATH': 'src/objective-c/examples/Sample', 'FRAMEWORKS': 'YES' - }), - self.config.job_spec( + }) + out += self.config.job_spec( ['src/objective-c/tests/build_one_example.sh'], timeout_seconds=10 * 60, - shortname='objc-build-example-switftsample', + shortname='ios-buildtest-example-switftsample', cpu_cost=1e6, environ={ 'SCHEME': 'SwiftSample', 'EXAMPLE_PATH': 'src/objective-c/examples/SwiftSample' - }), - self.config.job_spec( + }) + out += self.config.job_spec( + ['src/objective-c/tests/run_plugin_tests.sh'], + timeout_seconds=60 * 60, + shortname='ios-test-plugintest', + cpu_cost=1e6, + environ=_FORCE_ENVIRON_FOR_WRAPPERS) + out += self.config.job_spec( ['test/core/iomgr/ios/CFStreamTests/run_tests.sh'], timeout_seconds=20 * 60, - shortname='cfstream-tests', + shortname='ios-test-cfstream-tests', cpu_cost=1e6, - environ=_FORCE_ENVIRON_FOR_WRAPPERS), - ] + environ=_FORCE_ENVIRON_FOR_WRAPPERS) + out += self.config.job_spec( + ['src/objective-c/tests/run_one_test.sh'], + timeout_seconds=60 * 60, + shortname='ios-test-unittests', + cpu_cost=1e6, + environ={ + SCHEME: 'UnitTests' + }) + out += self.config.job_spec( + ['src/objective-c/tests/run_one_test.sh'], + timeout_seconds=60 * 60, + shortname='ios-test-interoptests', + cpu_cost=1e6, + environ={ + SCHEME: 'InteropTests' + }) + out += self.config.job_spec( + ['src/objective-c/tests/run_one_test.sh'], + timeout_seconds=60 * 60, + shortname='ios-test-cronettests', + cpu_cost=1e6, + environ={ + SCHEME: 'CronetTests' + }) + out += self.config.job_spec( + ['src/objective-c/tests/run_one_test.sh'], + timeout_seconds=60 * 60, + shortname='mac-test-basictests', + cpu_cost=1e6, + environ={ + SCHEME: 'MacTests' + }) + + return sorted(out) def pre_build_steps(self): return [] diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 785dff36eab..06a52b0f76e 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -246,7 +246,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): # supported on mac only. test_jobs += _generate_jobs( languages=['objc'], - configs=['dbg', 'opt'], + configs=['opt'], platforms=['macos'], labels=['basictests', 'multilang'], extra_args=extra_args, @@ -519,6 +519,12 @@ if __name__ == "__main__": type=str, nargs='?', help='Upload test results to a specified BQ table.') + argp.add_argument( + '--extra_args', + default='', + type=str, + nargs=argparse.REMAINDER, + help='Extra test args passed to each sub-script.') args = argp.parse_args() extra_args = [] @@ -536,6 +542,8 @@ if __name__ == "__main__": extra_args.append('--bq_result_table') extra_args.append('%s' % args.bq_result_table) extra_args.append('--measure_cpu_costs') + if args.extra_args: + extra_args.extend(args.extra_args) all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \ _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)