From 43628b286ff94de01aaee70f6578302db45d32ec Mon Sep 17 00:00:00 2001 From: Prashant Jaikumar Date: Wed, 19 Jun 2019 14:14:02 -0700 Subject: [PATCH 1/3] Update googletest version to v1.8.1 Bazel builds of test/cpp/end2end:end2end_test were failing on Mac OS with v1.8.0 due to missing gtest symbols. The issue is not seen in v1.8.1. A WORKSPACE file was added to gtest repo in 1.8.1, so gtest.BUILD can be removed. --- bazel/grpc_deps.bzl | 12 ++---- test/cpp/end2end/BUILD | 6 --- test/cpp/ext/filters/census/BUILD | 1 - test/cpp/naming/BUILD | 2 +- .../generate_resolver_component_tests.bzl | 4 +- test/cpp/server/load_reporter/BUILD | 1 - third_party/googletest | 2 +- third_party/gtest.BUILD | 42 ------------------- tools/run_tests/sanity/check_submodules.sh | 2 +- 9 files changed, 8 insertions(+), 64 deletions(-) delete mode 100644 third_party/gtest.BUILD diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 151825c6839..9512c25dfbe 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -66,11 +66,6 @@ def grpc_deps(): actual = "@com_github_google_googletest//:gtest", ) - native.bind( - name = "gmock", - actual = "@com_github_google_googletest//:gmock", - ) - native.bind( name = "benchmark", actual = "@com_github_google_benchmark//:benchmark", @@ -144,10 +139,9 @@ def grpc_deps(): if "com_github_google_googletest" not in native.existing_rules(): http_archive( name = "com_github_google_googletest", - build_file = "@com_github_grpc_grpc//third_party:gtest.BUILD", - sha256 = "175a22300b3450e27e5f2e6f95cc9abca74617cbc21a1e0ed19bdfbd22ea0305", - strip_prefix = "googletest-ec44c6c1675c25b9827aacd08c02433cccde7780", - url = "https://github.com/google/googletest/archive/ec44c6c1675c25b9827aacd08c02433cccde7780.tar.gz", + sha256 = "d0d447b4feeedca837a0d46a289d4223089b32ac2f84545fa4982755cc8919be", + strip_prefix = "googletest-2fe3bd994b3189899d93f1d5a881e725e046fdc2", + url = "https://github.com/google/googletest/archive/2fe3bd994b3189899d93f1d5a881e725e046fdc2.tar.gz", ) if "com_github_gflags_gflags" not in native.existing_rules(): diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index d229cc33034..ab0ef0c46ba 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -369,7 +369,6 @@ grpc_cc_test( name = "mock_test", srcs = ["mock_test.cc"], external_deps = [ - "gmock", "gtest", ], deps = [ @@ -406,7 +405,6 @@ grpc_cc_test( name = "client_lb_end2end_test", srcs = ["client_lb_end2end_test.cc"], external_deps = [ - "gmock", "gtest", ], deps = [ @@ -427,7 +425,6 @@ grpc_cc_test( name = "service_config_end2end_test", srcs = ["service_config_end2end_test.cc"], external_deps = [ - "gmock", "gtest", ], deps = [ @@ -447,7 +444,6 @@ grpc_cc_test( name = "grpclb_end2end_test", srcs = ["grpclb_end2end_test.cc"], external_deps = [ - "gmock", "gtest", ], deps = [ @@ -469,7 +465,6 @@ grpc_cc_test( name = "xds_end2end_test", srcs = ["xds_end2end_test.cc"], external_deps = [ - "gmock", "gtest", ], deps = [ @@ -594,7 +589,6 @@ grpc_cc_test( srcs = ["server_load_reporting_end2end_test.cc"], external_deps = [ "gtest", - "gmock", ], deps = [ "//:grpcpp_server_load_reporting", diff --git a/test/cpp/ext/filters/census/BUILD b/test/cpp/ext/filters/census/BUILD index 78b27e2063c..452a84b2a7a 100644 --- a/test/cpp/ext/filters/census/BUILD +++ b/test/cpp/ext/filters/census/BUILD @@ -26,7 +26,6 @@ grpc_cc_test( ], external_deps = [ "gtest", - "gmock", "opencensus-stats-test", ], language = "C++", diff --git a/test/cpp/naming/BUILD b/test/cpp/naming/BUILD index 7db435a3fd4..b91c0b83c9b 100644 --- a/test/cpp/naming/BUILD +++ b/test/cpp/naming/BUILD @@ -37,7 +37,7 @@ grpc_py_binary( grpc_cc_test( name = "cancel_ares_query_test", srcs = ["cancel_ares_query_test.cc"], - external_deps = ["gmock"], + external_deps = ["gtest"], deps = [ ":dns_test_util", "//:gpr", diff --git a/test/cpp/naming/generate_resolver_component_tests.bzl b/test/cpp/naming/generate_resolver_component_tests.bzl index bcc62f62871..2ce61d05446 100755 --- a/test/cpp/naming/generate_resolver_component_tests.bzl +++ b/test/cpp/naming/generate_resolver_component_tests.bzl @@ -23,7 +23,7 @@ def generate_resolver_component_tests(): "address_sorting_test.cc", ], external_deps = [ - "gmock", + "gtest", ], deps = [ "//test/cpp/util:test_util%s" % unsecure_build_config_suffix, @@ -43,7 +43,7 @@ def generate_resolver_component_tests(): "resolver_component_test.cc", ], external_deps = [ - "gmock", + "gtest", ], deps = [ ":dns_test_util", diff --git a/test/cpp/server/load_reporter/BUILD b/test/cpp/server/load_reporter/BUILD index db5c93263ad..633b6d8c0d9 100644 --- a/test/cpp/server/load_reporter/BUILD +++ b/test/cpp/server/load_reporter/BUILD @@ -35,7 +35,6 @@ grpc_cc_test( srcs = ["load_reporter_test.cc"], external_deps = [ "gtest", - "gmock", "opencensus-stats-test", ], deps = [ diff --git a/third_party/googletest b/third_party/googletest index ec44c6c1675..2fe3bd994b3 160000 --- a/third_party/googletest +++ b/third_party/googletest @@ -1 +1 @@ -Subproject commit ec44c6c1675c25b9827aacd08c02433cccde7780 +Subproject commit 2fe3bd994b3189899d93f1d5a881e725e046fdc2 diff --git a/third_party/gtest.BUILD b/third_party/gtest.BUILD deleted file mode 100644 index f38bfd8d457..00000000000 --- a/third_party/gtest.BUILD +++ /dev/null @@ -1,42 +0,0 @@ -cc_library( - name = "gtest", - srcs = [ - "googletest/src/gtest-all.cc", - ], - hdrs = glob([ - "googletest/include/**/*.h", - "googletest/src/*.cc", - "googletest/src/*.h", - ]), - includes = [ - "googletest", - "googletest/include", - ], - linkstatic = 1, - visibility = [ - "//visibility:public", - ], -) - -cc_library( - name = "gmock", - srcs = [ - "googlemock/src/gmock-all.cc" - ], - hdrs = glob([ - "googlemock/include/**/*.h", - "googlemock/src/*.cc", - "googlemock/src/*.h" - ]), - includes = [ - "googlemock", - "googlemock/include", - ], - deps = [ - ":gtest", - ], - linkstatic = 1, - visibility = [ - "//visibility:public", - ], -) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 487479ba3da..7b11d49655c 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -35,7 +35,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" 911001cdca003337bdb93fab32740cde61bafee3 third_party/data-plane-api (heads/master) 28f50e0fed19872e0fd50dd23ce2ee8cd759338e third_party/gflags (v2.2.0-5-g30dbc81) 80ed4d0bbf65d57cc267dfc63bd2584557f11f9b third_party/googleapis (common-protos-1_3_1-915-g80ed4d0bb) - ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) + 2fe3bd994b3189899d93f1d5a881e725e046fdc2 third_party/googletest (release-1.8.1) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) 09745575a923640154bcf307fba8aedff47f240a third_party/protobuf (v3.7.0-rc.2-247-g09745575) From 624839b7043e8ac3463765152e1d9326d474c691 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 20 Jun 2019 10:32:54 -0700 Subject: [PATCH 2/3] Add example Python server using compression. --- examples/python/compression/BUILD.bazel | 44 +++++++ examples/python/compression/README.md | 58 ++++++++++ examples/python/compression/client.py | 76 ++++++++++++ examples/python/compression/server.py | 109 ++++++++++++++++++ .../test/compression_example_test.py | 62 ++++++++++ 5 files changed, 349 insertions(+) create mode 100644 examples/python/compression/BUILD.bazel create mode 100644 examples/python/compression/README.md create mode 100644 examples/python/compression/client.py create mode 100644 examples/python/compression/server.py create mode 100644 examples/python/compression/test/compression_example_test.py diff --git a/examples/python/compression/BUILD.bazel b/examples/python/compression/BUILD.bazel new file mode 100644 index 00000000000..b95d7cccc77 --- /dev/null +++ b/examples/python/compression/BUILD.bazel @@ -0,0 +1,44 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +py_binary( + name = "server", + srcs = ["server.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//examples:py_helloworld", + ], + srcs_version = "PY2AND3", +) + +py_binary( + name = "client", + srcs = ["client.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//examples:py_helloworld", + ], + srcs_version = "PY2AND3", +) + +py_test( + name = "test/compression_example_test", + srcs = ["test/compression_example_test.py"], + srcs_version = "PY2AND3", + data = [ + ":client", + ":server", + ], + size = "small", +) diff --git a/examples/python/compression/README.md b/examples/python/compression/README.md new file mode 100644 index 00000000000..c719bba07f8 --- /dev/null +++ b/examples/python/compression/README.md @@ -0,0 +1,58 @@ +## Compression with gRPC Python + +gRPC offers lossless compression options in order to decrease the number of bits +transferred over the wire. Three levels of compression are available: + + - `grpc.Compression.NoCompression` - No compression is applied to the payload. (default) + - `grpc.Compression.Deflate` - The "Deflate" algorithm is applied to the payload. + - `grpc.Compression.Gzip` - The Gzip algorithm is applied to the payload. + +The default option on both clients and servers is `grpc.Compression.NoCompression`. + +See [the gRPC Compression Spec](https://github.com/grpc/grpc/blob/master/doc/compression.md) +for more information. + +### Client Side Compression + +Compression may be set at two levels on the client side. + +#### At the channel level + +```python +with grpc.insecure_channel('foo.bar:1234', compression=grpc.Compression.Gzip) as channel: + use_channel(channel) +``` + +#### At the call level + +Setting the compression method at the call level will override any settings on +the channel level. + +```python +stub = helloworld_pb2_grpc.GreeterStub(channel) +response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'), + compression=grpc.Compression.Deflate) +``` + + +### Server Side Compression + +Additionally, compression may be set at two levels on the server side. + +#### On the entire server + +```python +server = grpc.server(futures.ThreadPoolExecutor(), + compression=grpc.Compression.Gzip) +``` + +#### For an individual RPC + +```python +def SayHello(self, request, context): + context.set_response_compression(grpc.Compression.NoCompression) + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) +``` + +Setting the compression method for an individual RPC will override any setting +supplied at server creation time. diff --git a/examples/python/compression/client.py b/examples/python/compression/client.py new file mode 100644 index 00000000000..444f14c1f68 --- /dev/null +++ b/examples/python/compression/client.py @@ -0,0 +1,76 @@ +# Copyright 2019 the gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example of compression on the client side with gRPC.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import logging +import grpc + +from examples import helloworld_pb2 +from examples import helloworld_pb2_grpc + +_DESCRIPTION = 'A client capable of compression.' +_COMPRESSION_OPTIONS = { + "none": grpc.Compression.NoCompression, + "deflate": grpc.Compression.Deflate, + "gzip": grpc.Compression.Gzip, +} + +_LOGGER = logging.getLogger(__name__) + + +def run_client(channel_compression, call_compression, target): + with grpc.insecure_channel( + target, compression=channel_compression) as channel: + stub = helloworld_pb2_grpc.GreeterStub(channel) + response = stub.SayHello( + helloworld_pb2.HelloRequest(name='you'), + compression=call_compression, + wait_for_ready=True) + print("Response: {}".format(response)) + + +def main(): + parser = argparse.ArgumentParser(description=_DESCRIPTION) + parser.add_argument( + '--channel_compression', + default='none', + nargs='?', + choices=_COMPRESSION_OPTIONS.keys(), + help='The compression method to use for the channel.') + parser.add_argument( + '--call_compression', + default='none', + nargs='?', + choices=_COMPRESSION_OPTIONS.keys(), + help='The compression method to use for an individual call.') + parser.add_argument( + '--server', + default='localhost:50051', + type=str, + nargs='?', + help='The host-port pair at which to reach the server.') + args = parser.parse_args() + channel_compression = _COMPRESSION_OPTIONS[args.channel_compression] + call_compression = _COMPRESSION_OPTIONS[args.call_compression] + run_client(channel_compression, call_compression, args.server) + + +if __name__ == "__main__": + logging.basicConfig() + main() diff --git a/examples/python/compression/server.py b/examples/python/compression/server.py new file mode 100644 index 00000000000..bc13e60cb5b --- /dev/null +++ b/examples/python/compression/server.py @@ -0,0 +1,109 @@ +# Copyright 2019 the gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example of compression on the server side with gRPC.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from concurrent import futures +import argparse +import logging +import threading +import time +import grpc + +from examples import helloworld_pb2 +from examples import helloworld_pb2_grpc + +_DESCRIPTION = 'A server capable of compression.' +_COMPRESSION_OPTIONS = { + "none": grpc.Compression.NoCompression, + "deflate": grpc.Compression.Deflate, + "gzip": grpc.Compression.Gzip, +} +_LOGGER = logging.getLogger(__name__) + +_SERVER_HOST = 'localhost' +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def __init__(self, no_compress_every_n): + super(Greeter, self).__init__() + self._no_compress_every_n = 0 + self._request_counter = 0 + self._counter_lock = threading.RLock() + + def _should_suppress_compression(self): + suppress_compression = False + with self._counter_lock: + if self._no_compress_every_n and self._request_counter % self._no_compress_every_n == 0: + suppress_compression = True + self._request_counter += 1 + return suppress_compression + + def SayHello(self, request, context): + if self._should_suppress_compression(): + context.set_response_compression(grpc.Compression.NoCompression) + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + +def run_server(server_compression, no_compress_every_n, port): + server = grpc.server( + futures.ThreadPoolExecutor(), + compression=server_compression, + options=(('grpc.so_reuseport', 1),)) + helloworld_pb2_grpc.add_GreeterServicer_to_server( + Greeter(no_compress_every_n), server) + address = '{}:{}'.format(_SERVER_HOST, port) + server.add_insecure_port(address) + server.start() + print("Server listening at '{}'".format(address)) + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(None) + + +def main(): + parser = argparse.ArgumentParser(description=_DESCRIPTION) + parser.add_argument( + '--server_compression', + default='none', + nargs='?', + choices=_COMPRESSION_OPTIONS.keys(), + help='The default compression method for the server.') + parser.add_argument( + '--no_compress_every_n', + type=int, + default=0, + nargs='?', + help='If set, every nth reply will be uncompressed.') + parser.add_argument( + '--port', + type=int, + default=50051, + nargs='?', + help='The port on which the server will listen.') + args = parser.parse_args() + run_server(_COMPRESSION_OPTIONS[args.server_compression], + args.no_compress_every_n, args.port) + + +if __name__ == "__main__": + logging.basicConfig() + main() diff --git a/examples/python/compression/test/compression_example_test.py b/examples/python/compression/test/compression_example_test.py new file mode 100644 index 00000000000..7d25379683f --- /dev/null +++ b/examples/python/compression/test/compression_example_test.py @@ -0,0 +1,62 @@ +# Copyright 2019 the gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test for compression example.""" + +import contextlib +import os +import socket +import subprocess +import unittest + +_BINARY_DIR = os.path.realpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +_SERVER_PATH = os.path.join(_BINARY_DIR, 'server') +_CLIENT_PATH = os.path.join(_BINARY_DIR, 'client') + + +@contextlib.contextmanager +def _get_port(): + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 0: + raise RuntimeError("Failed to set SO_REUSEPORT.") + sock.bind(('', 0)) + try: + yield sock.getsockname()[1] + finally: + sock.close() + + +class CompressionExampleTest(unittest.TestCase): + + def test_compression_example(self): + with _get_port() as test_port: + server_process = subprocess.Popen((_SERVER_PATH, '--port', + str(test_port), + '--server_compression', 'gzip')) + try: + server_target = 'localhost:{}'.format(test_port) + client_process = subprocess.Popen( + (_CLIENT_PATH, '--server', server_target, + '--channel_compression', 'gzip')) + client_return_code = client_process.wait() + self.assertEqual(0, client_return_code) + self.assertIsNone(server_process.poll()) + finally: + server_process.kill() + server_process.wait() + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 8fb51946bf1a14039f5aab9abeb4144008dfb1e9 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 21 Jun 2019 14:01:27 -0700 Subject: [PATCH 3/3] Fix multiprocessing example for MacOS. A closer reading of the API for getsockopt revealed that we were depending on an implementation detail of getsockopt on Linux. This assumption breaks down on MacOS. getsockopt merely guarantees that it will return on 0 in case of failure and a value greater than 0 in case of success. There is no guarantee as to *which* non-zero value you will receive. On Linux, it seems to be 1, the value which was explicitly set. On MacOS, it seems to be the value of the FLAG which was set, i.e. 512 for SO_REUSEPORT. This commit ensures the check we use does not rely on either of these implementation details. --- examples/python/multiprocessing/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/multiprocessing/server.py b/examples/python/multiprocessing/server.py index a05eb9edda0..b1e5951a8b0 100644 --- a/examples/python/multiprocessing/server.py +++ b/examples/python/multiprocessing/server.py @@ -87,7 +87,7 @@ def _reserve_port(): """Find and reserve a port for all subprocesses to use.""" sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) != 1: + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 0: raise RuntimeError("Failed to set SO_REUSEPORT.") sock.bind(('', 0)) try: