diff --git a/BUILD b/BUILD index 82ac73f0147..13843195ea7 100644 --- a/BUILD +++ b/BUILD @@ -38,6 +38,11 @@ config_setting( values = {"define": "grpc_no_ares=true"}, ) +config_setting( + name = "remote_execution", + values = {"define": "GRPC_PORT_ISOLATED_RUNTIME=1"}, +) + # This should be updated along with build.yaml g_stands_for = "generous" diff --git a/CMakeLists.txt b/CMakeLists.txt index ab57f662cf2..921487e6f41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1621,6 +1621,7 @@ add_library(grpc_test_util test/core/util/parse_hexstring.cc test/core/util/passthru_endpoint.cc test/core/util/port.cc + test/core/util/port_isolated_runtime_environment.cc test/core/util/port_server_client.cc test/core/util/slice_splitter.cc test/core/util/tracer_util.cc @@ -1890,6 +1891,7 @@ add_library(grpc_test_util_unsecure test/core/util/parse_hexstring.cc test/core/util/passthru_endpoint.cc test/core/util/port.cc + test/core/util/port_isolated_runtime_environment.cc test/core/util/port_server_client.cc test/core/util/slice_splitter.cc test/core/util/tracer_util.cc diff --git a/Makefile b/Makefile index bd83244a616..0e52d5a3254 100644 --- a/Makefile +++ b/Makefile @@ -3627,6 +3627,7 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/util/parse_hexstring.cc \ test/core/util/passthru_endpoint.cc \ test/core/util/port.cc \ + test/core/util/port_isolated_runtime_environment.cc \ test/core/util/port_server_client.cc \ test/core/util/slice_splitter.cc \ test/core/util/tracer_util.cc \ @@ -3887,6 +3888,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/util/parse_hexstring.cc \ test/core/util/passthru_endpoint.cc \ test/core/util/port.cc \ + test/core/util/port_isolated_runtime_environment.cc \ test/core/util/port_server_client.cc \ test/core/util/slice_splitter.cc \ test/core/util/tracer_util.cc \ diff --git a/WORKSPACE b/WORKSPACE index cead0bc877b..bf09aa7cc93 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -115,3 +115,13 @@ http_archive( strip_prefix = "abseil-cpp-cc4bed2d74f7c8717e31f9579214ab52a9c9c610", url = "https://github.com/abseil/abseil-cpp/archive/cc4bed2d74f7c8717e31f9579214ab52a9c9c610.tar.gz", ) + +http_archive( + name = "bazel_toolchains", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz", + ], + strip_prefix = "bazel-toolchains-af4681c3d19f063f090222ec3d04108c4e0ca255", + sha256 = "d58bb2d6c8603f600d522b6104d6192a65339aa26cbba9f11ff5c4b36dedb928", +) diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index b35ca73745f..2ee8c297da1 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -33,10 +33,10 @@ def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [], native.cc_library( name = name, srcs = srcs, - defines = select({ - "//:grpc_no_ares": ["GRPC_ARES=0"], - "//conditions:default": [], - }), + defines = select({"//:grpc_no_ares": ["GRPC_ARES=0"], + "//conditions:default": [],}) + + select({"//:remote_execution": ["GRPC_PORT_ISOLATED_RUNTIME=1"], + "//conditions:default": [],}), hdrs = hdrs + public_hdrs, deps = deps + ["//external:" + dep for dep in external_deps], copts = copts, diff --git a/build.yaml b/build.yaml index 40c37dc53f7..854a695acce 100644 --- a/build.yaml +++ b/build.yaml @@ -736,6 +736,7 @@ filegroups: - test/core/util/parse_hexstring.cc - test/core/util/passthru_endpoint.cc - test/core/util/port.cc + - test/core/util/port_isolated_runtime_environment.cc - test/core/util/port_server_client.cc - test/core/util/slice_splitter.cc - test/core/util/tracer_util.cc diff --git a/grpc.gyp b/grpc.gyp index 4ceb480282c..c34206b1a5d 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -511,6 +511,7 @@ 'test/core/util/parse_hexstring.cc', 'test/core/util/passthru_endpoint.cc', 'test/core/util/port.cc', + 'test/core/util/port_isolated_runtime_environment.cc', 'test/core/util/port_server_client.cc', 'test/core/util/slice_splitter.cc', 'test/core/util/tracer_util.cc', @@ -722,6 +723,7 @@ 'test/core/util/parse_hexstring.cc', 'test/core/util/passthru_endpoint.cc', 'test/core/util/port.cc', + 'test/core/util/port_isolated_runtime_environment.cc', 'test/core/util/port_server_client.cc', 'test/core/util/slice_splitter.cc', 'test/core/util/tracer_util.cc', diff --git a/test/core/util/BUILD b/test/core/util/BUILD index 268547f6c9d..ec88d767ce6 100644 --- a/test/core/util/BUILD +++ b/test/core/util/BUILD @@ -57,6 +57,7 @@ grpc_cc_library( "parse_hexstring.cc", "passthru_endpoint.cc", "port.cc", + "port_isolated_runtime_environment.cc", "port_server_client.cc", "reconnect_server.cc", "slice_splitter.cc", diff --git a/test/core/util/port_isolated_runtime_environment.cc b/test/core/util/port_isolated_runtime_environment.cc new file mode 100644 index 00000000000..5f0585e9fb4 --- /dev/null +++ b/test/core/util/port_isolated_runtime_environment.cc @@ -0,0 +1,42 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* When running tests on remote machines, the framework takes a round-robin pick + * of a port within certain range. There is no need to recycle ports. + */ +#include "src/core/lib/iomgr/port.h" +#include "test/core/util/test_config.h" +#if defined(GRPC_PORT_ISOLATED_RUNTIME) + +#include "test/core/util/port.h" + +#define LOWER_PORT 49152 +static int s_allocated_port = LOWER_PORT; + +int grpc_pick_unused_port_or_die(void) { + int allocated_port = s_allocated_port++; + if (s_allocated_port == 65536) { + s_allocated_port = LOWER_PORT; + } + + return allocated_port; +} + +void grpc_recycle_unused_port(int port) { (void)port; } + +#endif /* GRPC_PORT_ISOLATED_RUNTIME */ diff --git a/test/core/util/test_config.h b/test/core/util/test_config.h index 775ffac9497..5b3d34799ee 100644 --- a/test/core/util/test_config.h +++ b/test/core/util/test_config.h @@ -33,7 +33,7 @@ gpr_timespec grpc_timeout_seconds_to_deadline(int64_t time_s); /* Converts a given timeout (in milliseconds) to a deadline. */ gpr_timespec grpc_timeout_milliseconds_to_deadline(int64_t time_ms); -#ifndef GRPC_TEST_CUSTOM_PICK_PORT +#if !defined(GRPC_TEST_CUSTOM_PICK_PORT) && !defined(GRPC_PORT_ISOLATED_RUNTIME) #define GRPC_TEST_PICK_PORT #endif diff --git a/third_party/zlib.BUILD b/third_party/zlib.BUILD index 7879a81c680..a71c85fa98d 100644 --- a/third_party/zlib.BUILD +++ b/third_party/zlib.BUILD @@ -27,7 +27,7 @@ cc_library( "zutil.h", ], includes = [ - "include", + ".", ], linkstatic = 1, visibility = [ diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry.sh new file mode 100644 index 00000000000..e328be8aaba --- /dev/null +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# A temporary solution to give Kokoro credentials. +# The file name 4321_grpc-testing-service needs to match auth_credential in +# the build config. +mkdir -p ${KOKORO_KEYSTORE_DIR} +cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service + +mkdir -p /tmpfs/tmp/bazel-canary +ln -f "${KOKORO_GFILE_DIR}/bazel-canary" /tmpfs/tmp/bazel-canary/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary" +export PATH="/tmpfs/tmp/bazel-canary:${PATH}" +# This should show /tmpfs/tmp/bazel-canary/bazel +which bazel +chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py" + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ + --host_jvm_args=-Dbazel.DigestFunction=SHA1 \ + test --jobs="50" \ + --test_timeout="300,450,1200,3600" \ + --test_output=errors \ + --verbose_failures=true \ + --keep_going \ + --remote_accept_cached=true \ + --spawn_strategy=remote \ + --remote_local_fallback=false \ + --remote_timeout=3600 \ + --strategy=Javac=remote \ + --strategy=Closure=remote \ + --genrule_strategy=remote \ + --experimental_strict_action_env=true \ + --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-fl@sha256:aa20628a902f06a11a015caa94b0432eb60690de2d2525bd046b9eea046f5d8a" }' \ + --crosstool_top=@bazel_toolchains//configs/debian8_clang/0.2.0/bazel_0.7.0:toolchain \ + --define GRPC_PORT_ISOLATED_RUNTIME=1 \ + -- //test/... diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index a0ea4238e1e..ae7ddb24751 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8966,6 +8966,7 @@ "test/core/util/passthru_endpoint.h", "test/core/util/port.cc", "test/core/util/port.h", + "test/core/util/port_isolated_runtime_environment.cc", "test/core/util/port_server_client.cc", "test/core/util/port_server_client.h", "test/core/util/slice_splitter.cc",