Merge branch 'master' into out_of_bounds_tests

pull/19359/head
Yash Tibrewal 6 years ago
commit 24c232009c
  1. 59
      examples/python/debug/BUILD.bazel
  2. 68
      examples/python/debug/README.md
  3. 90
      examples/python/debug/debug_server.py
  4. 49
      examples/python/debug/get_stats.py
  5. 64
      examples/python/debug/send_message.py
  6. 53
      examples/python/debug/test/_debug_example_test.py
  7. 11
      src/csharp/Grpc.Tools/build/_grpc/Grpc.CSharp.xml
  8. 3
      src/csharp/Grpc.Tools/build/_grpc/_Grpc.Tools.targets
  9. 5
      src/objective-c/tests/InteropTests/InteropTests.m
  10. 5
      src/objective-c/tests/MacTests/StressTests.m
  11. 22
      src/objective-c/tests/examples_build_test.sh
  12. 3
      test/core/bad_client/gen_build_yaml.py
  13. 6
      test/core/bad_client/generate_tests.bzl
  14. 65
      test/cpp/microbenchmarks/bm_chttp2_transport.cc

@ -0,0 +1,59 @@
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("@grpc_python_dependencies//:requirements.bzl", "requirement")
py_binary(
name = "debug_server",
testonly = 1,
srcs = ["debug_server.py"],
deps = [
"//src/python/grpcio/grpc:grpcio",
"//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz",
"//examples:py_helloworld",
],
)
py_binary(
name = "send_message",
testonly = 1,
srcs = ["send_message.py"],
deps = [
"//src/python/grpcio/grpc:grpcio",
"//examples:py_helloworld",
],
)
py_binary(
name = "get_stats",
testonly = 1,
srcs = ["get_stats.py"],
deps = [
"//src/python/grpcio/grpc:grpcio",
"//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz",
],
)
py_test(
name = "_debug_example_test",
srcs = ["test/_debug_example_test.py"],
deps = [
"//src/python/grpcio/grpc:grpcio",
"//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz",
"//examples:py_helloworld",
":debug_server",
":send_message",
":get_stats",
],
)

@ -0,0 +1,68 @@
# gRPC Python Debug Example
This example demonstrate the usage of Channelz. For a better looking website,
the [gdebug](https://github.com/grpc/grpc-experiments/tree/master/gdebug) uses
gRPC-Web protocol and will serve all useful information in web pages.
## Channelz: Live Channel Tracing
Channelz is a channel tracing feature. It will track statistics like how many
messages have been sent, how many of them failed, what are the connected
sockets. Since it is implemented in C-Core and has low-overhead, it is
recommended to turn on for production services. See [Channelz design
doc](https://github.com/grpc/proposal/blob/master/A14-channelz.md).
## How to enable tracing log
The tracing log generation might have larger overhead, especially when you try
to trace transport. It would result in replicating the traffic loads. However,
it is still the most powerful tool when you need to dive in.
### The Most Verbose Tracing Log
Specify environment variables, then run your application:
```
GRPC_VERBOSITY=debug
GRPC_TRACE=all
```
For more granularity, please see
[environment_variables](https://github.com/grpc/grpc/blob/master/doc/environment_variables.md).
### Debug Transport Protocol
```
GRPC_VERBOSITY=debug
GRPC_TRACE=tcp,http,secure_endpoint,transport_security
```
### Debug Connection Behavior
```
GRPC_VERBOSITY=debug
GRPC_TRACE=call_error,connectivity_state,pick_first,round_robin,glb
```
## How to debug your application?
`pdb` is a debugging tool that is available for Python interpreters natively.
You can set breakpoint, and execute commands while the application is stopped.
The simplest usage is add a single line in the place you want to inspect:
`import pdb; pdb.set_trace()`. When interpreter see this line, it would pop out
a interactive command line interface for you to inspect the application state.
For more detailed usage, see https://docs.python.org/3/library/pdb.html.
**Caveat**: gRPC Python uses C-Extension under-the-hood, so `pdb` may not be
able to trace through the whole stack.
## gRPC Command Line Tool
`grpc_cli` is a handy tool to interact with gRPC backend easily. Imageine you can
inspect what service does a server provide without writing any code, and make
gRPC calls just like `curl`.
The installation guide: https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md#code-location
The usage guide: https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md#usage
The source code: https://github.com/grpc/grpc/blob/master/test/cpp/util/grpc_cli.cc

@ -0,0 +1,90 @@
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Python example of utilizing Channelz feature."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import logging
import time
from concurrent import futures
import random
import grpc
from grpc_channelz.v1 import channelz
from examples import helloworld_pb2
from examples import helloworld_pb2_grpc
_LOGGER = logging.getLogger(__name__)
_LOGGER.setLevel(logging.INFO)
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
_RANDOM_FAILURE_RATE = 0.3
class FaultInjectGreeter(helloworld_pb2_grpc.GreeterServicer):
def __init__(self, failure_rate):
self._failure_rate = failure_rate
def SayHello(self, request, context):
if random.random() < self._failure_rate:
context.abort(grpc.StatusCode.UNAVAILABLE,
'Randomly injected failure.')
return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
def create_server(addr, failure_rate):
server = grpc.server(futures.ThreadPoolExecutor())
helloworld_pb2_grpc.add_GreeterServicer_to_server(
FaultInjectGreeter(failure_rate), server)
# Add Channelz Servicer to the gRPC server
channelz.add_channelz_servicer(server)
server.add_insecure_port(addr)
return server
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--addr',
nargs=1,
type=str,
default='[::]:50051',
help='the address to listen on')
parser.add_argument(
'--failure_rate',
nargs=1,
type=float,
default=0.3,
help='a float indicates the percentage of failed message injections')
args = parser.parse_args()
server = create_server(addr=args.addr, failure_rate=args.failure_rate)
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
main()

@ -0,0 +1,49 @@
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Poll statistics from the server."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import argparse
import grpc
from grpc_channelz.v1 import channelz_pb2
from grpc_channelz.v1 import channelz_pb2_grpc
def run(addr):
with grpc.insecure_channel(addr) as channel:
channelz_stub = channelz_pb2_grpc.ChannelzStub(channel)
response = channelz_stub.GetServers(
channelz_pb2.GetServersRequest(start_server_id=0))
print('Info for all servers: %s' % response)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--addr',
nargs=1,
type=str,
default='[::]:50051',
help='the address to request')
args = parser.parse_args()
run(addr=args.addr)
if __name__ == '__main__':
logging.basicConfig()
main()

@ -0,0 +1,64 @@
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Send multiple greeting messages to the backend."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import argparse
import grpc
from examples import helloworld_pb2
from examples import helloworld_pb2_grpc
def process(stub, request):
try:
response = stub.SayHello(request)
except grpc.RpcError as rpc_error:
print('Received error: %s' % rpc_error)
else:
print('Received message: %s' % response)
def run(addr, n):
with grpc.insecure_channel(addr) as channel:
stub = helloworld_pb2_grpc.GreeterStub(channel)
request = helloworld_pb2.HelloRequest(name='you')
for _ in range(n):
process(stub, request)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--addr',
nargs=1,
type=str,
default='[::]:50051',
help='the address to request')
parser.add_argument(
'-n',
nargs=1,
type=int,
default=10,
help='an integer for number of messages to sent')
args = parser.parse_args()
run(addr=args.addr, n=args.n)
if __name__ == '__main__':
logging.basicConfig()
main()

@ -0,0 +1,53 @@
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test for gRPC Python debug example."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import unittest
from examples.python.debug import debug_server
from examples.python.debug import send_message
from examples.python.debug import get_stats
_LOGGER = logging.getLogger(__name__)
_LOGGER.setLevel(logging.INFO)
_FAILURE_RATE = 0.5
_NUMBER_OF_MESSAGES = 100
_ADDR_TEMPLATE = 'localhost:%d'
class DebugExampleTest(unittest.TestCase):
def test_channelz_example(self):
server = debug_server.create_server(
addr='[::]:0', failure_rate=_FAILURE_RATE)
port = server.add_insecure_port('[::]:0')
server.start()
address = _ADDR_TEMPLATE % port
send_message.run(addr=address, n=_NUMBER_OF_MESSAGES)
get_stats.run(addr=address)
server.stop(None)
# No unhandled exception raised, test passed!
if __name__ == '__main__':
logging.basicConfig()
unittest.main(verbosity=2)

@ -26,5 +26,16 @@
</EnumProperty.DataSource>
</EnumProperty>
<EnumProperty Name="ClientBaseType" DisplayName="gRPC Client Base Type"
Category="gRPC" Default="ClientBase"
Description="The base type to use for the client. This is an experimental feature.">
<EnumValue Name="ClientBase" DisplayName="Use ClientBase" />
<EnumValue Name="LiteClientBase" DisplayName="Use LiteClientBase" />
<EnumProperty.DataSource>
<DataSource ItemType="Protobuf" SourceOfDefaultValue="AfterContext"
PersistenceStyle="Attribute" />
</EnumProperty.DataSource>
</EnumProperty>
</Rule>
</ProjectSchemaDefinitions>

@ -42,6 +42,9 @@
<Protobuf_Compile Condition=" '%(Protobuf_Compile.GrpcServices)' == 'Server' ">
<_GrpcOutputOptions>%(Protobuf_Compile._GrpcOutputOptions);no_client</_GrpcOutputOptions>
</Protobuf_Compile>
<Protobuf_Compile Condition=" '%(Protobuf_Compile.GrpcServices)' == 'Client' or '%(Protobuf_Compile.GrpcServices)' == 'Both' ">
<_GrpcOutputOptions Condition=" '%(Protobuf_Compile.ClientBaseType)' == 'LiteClientBase' ">%(Protobuf_Compile._GrpcOutputOptions);lite_client</_GrpcOutputOptions>
</Protobuf_Compile>
</ItemGroup>
</Target>
</Project>

@ -364,6 +364,11 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager
[GRPCCall resetHostSettings];
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
[GRPCCall closeOpenConnections];
#pragma clang diagnostic pop
_service = [[self class] host] ? [RMTTestService serviceWithHost:[[self class] host]] : nil;
}

@ -118,6 +118,11 @@ extern const char *kCFStreamVarName;
[GRPCCall resetHostSettings];
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
[GRPCCall closeOpenConnections];
#pragma clang diagnostic pop
GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
options.transportType = [[self class] transportType];
options.PEMRootCertificates = [[self class] PEMRootCertificates];

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2019 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is used after a release to verify the released pods are working appropriately
set -ex
SCHEME=HelloWorld EXAMPLE_PATH=examples/objective-c/helloworld ./build_one_example.sh
SCHEME=RouteGuideClient EXAMPLE_PATH=examples/objective-c/route_guide ./build_one_example.sh
SCHEME=AuthSample EXAMPLE_PATH=examples/objective-c/auth_sample ./build_one_example.sh

@ -17,6 +17,7 @@
"""Generates the appropriate build.json data for all the bad_client tests."""
from __future__ import print_function
import collections
import yaml
@ -78,7 +79,7 @@ def main():
]
}
for t in sorted(BAD_CLIENT_TESTS.keys())]}
print yaml.dump(json)
print(yaml.dump(json))
if __name__ == '__main__':

@ -43,9 +43,6 @@ def grpc_bad_client_tests():
name = 'bad_client_test',
srcs = ['bad_client.cc'],
hdrs = ['bad_client.h'],
external_deps = [
"gtest",
],
language = "C++",
deps = ['//test/core/util:grpc_test_util', '//:grpc', '//:gpr', '//test/core/end2end:cq_verifier']
)
@ -54,5 +51,8 @@ def grpc_bad_client_tests():
name = '%s_bad_client_test' % t,
srcs = ['tests/%s.cc' % t],
deps = [':bad_client_test'],
external_deps = [
"gtest",
],
)

@ -259,18 +259,21 @@ static void BM_StreamCreateDestroy(benchmark::State& state) {
TrackCounters track_counters;
grpc_core::ExecCtx exec_ctx;
Fixture f(grpc::ChannelArguments(), true);
Stream s(&f);
auto* s = new Stream(&f);
grpc_transport_stream_op_batch op;
grpc_transport_stream_op_batch_payload op_payload(nullptr);
memset(&op, 0, sizeof(op));
op.cancel_stream = true;
op.payload = &op_payload;
op_payload.cancel_stream.cancel_error = GRPC_ERROR_CANCELLED;
std::unique_ptr<Closure> next = MakeClosure([&](grpc_error* error) {
if (!state.KeepRunning()) return;
s.Init(state);
s.Op(&op);
s.DestroyThen(next.get());
std::unique_ptr<Closure> next = MakeClosure([&, s](grpc_error* error) {
if (!state.KeepRunning()) {
delete s;
return;
}
s->Init(state);
s->Op(&op);
s->DestroyThen(next.get());
});
GRPC_CLOSURE_RUN(next.get(), GRPC_ERROR_NONE);
f.FlushExecCtx();
@ -305,7 +308,7 @@ static void BM_StreamCreateSendInitialMetadataDestroy(benchmark::State& state) {
TrackCounters track_counters;
grpc_core::ExecCtx exec_ctx;
Fixture f(grpc::ChannelArguments(), true);
Stream s(&f);
auto* s = new Stream(&f);
grpc_transport_stream_op_batch op;
grpc_transport_stream_op_batch_payload op_payload(nullptr);
std::unique_ptr<Closure> start;
@ -327,21 +330,24 @@ static void BM_StreamCreateSendInitialMetadataDestroy(benchmark::State& state) {
}
f.FlushExecCtx();
start = MakeClosure([&](grpc_error* error) {
if (!state.KeepRunning()) return;
s.Init(state);
start = MakeClosure([&, s](grpc_error* error) {
if (!state.KeepRunning()) {
delete s;
return;
}
s->Init(state);
reset_op();
op.on_complete = done.get();
op.send_initial_metadata = true;
op.payload->send_initial_metadata.send_initial_metadata = &b;
s.Op(&op);
s->Op(&op);
});
done = MakeClosure([&](grpc_error* error) {
reset_op();
op.cancel_stream = true;
op.payload->cancel_stream.cancel_error = GRPC_ERROR_CANCELLED;
s.Op(&op);
s.DestroyThen(start.get());
s->Op(&op);
s->DestroyThen(start.get());
});
GRPC_CLOSURE_SCHED(start.get(), GRPC_ERROR_NONE);
f.FlushExecCtx();
@ -355,8 +361,8 @@ static void BM_TransportEmptyOp(benchmark::State& state) {
TrackCounters track_counters;
grpc_core::ExecCtx exec_ctx;
Fixture f(grpc::ChannelArguments(), true);
Stream s(&f);
s.Init(state);
auto* s = new Stream(&f);
s->Init(state);
grpc_transport_stream_op_batch op;
grpc_transport_stream_op_batch_payload op_payload(nullptr);
auto reset_op = [&]() {
@ -367,15 +373,15 @@ static void BM_TransportEmptyOp(benchmark::State& state) {
if (!state.KeepRunning()) return;
reset_op();
op.on_complete = c.get();
s.Op(&op);
s->Op(&op);
});
GRPC_CLOSURE_SCHED(c.get(), GRPC_ERROR_NONE);
f.FlushExecCtx();
reset_op();
op.cancel_stream = true;
op_payload.cancel_stream.cancel_error = GRPC_ERROR_CANCELLED;
s.Op(&op);
s.DestroyThen(MakeOnceClosure([](grpc_error* error) {}));
s->Op(&op);
s->DestroyThen(MakeOnceClosure([s](grpc_error* error) { delete s; }));
f.FlushExecCtx();
track_counters.Finish(state);
}
@ -387,7 +393,7 @@ static void BM_TransportStreamSend(benchmark::State& state) {
TrackCounters track_counters;
grpc_core::ExecCtx exec_ctx;
Fixture f(grpc::ChannelArguments(), true);
auto s = std::unique_ptr<Stream>(new Stream(&f));
auto* s = new Stream(&f);
s->Init(state);
grpc_transport_stream_op_batch op;
grpc_transport_stream_op_batch_payload op_payload(nullptr);
@ -450,9 +456,8 @@ static void BM_TransportStreamSend(benchmark::State& state) {
op.cancel_stream = true;
op.payload->cancel_stream.cancel_error = GRPC_ERROR_CANCELLED;
s->Op(&op);
s->DestroyThen(MakeOnceClosure([](grpc_error* error) {}));
s->DestroyThen(MakeOnceClosure([s](grpc_error* error) { delete s; }));
f.FlushExecCtx();
s.reset();
track_counters.Finish(state);
grpc_metadata_batch_destroy(&b);
grpc_slice_unref(send_slice);
@ -520,8 +525,8 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
TrackCounters track_counters;
grpc_core::ExecCtx exec_ctx;
Fixture f(grpc::ChannelArguments(), true);
Stream s(&f);
s.Init(state);
auto* s = new Stream(&f);
s->Init(state);
grpc_transport_stream_op_batch_payload op_payload(nullptr);
grpc_transport_stream_op_batch op;
grpc_core::OrphanablePtr<grpc_core::ByteStream> recv_stream;
@ -557,7 +562,7 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
std::unique_ptr<Closure> c = MakeClosure([&](grpc_error* error) {
if (!state.KeepRunning()) return;
// force outgoing window to be yuge
s.chttp2_stream()->flow_control->TestOnlyForceHugeWindow();
s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow();
f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow();
received = 0;
reset_op();
@ -565,7 +570,7 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
op.recv_message = true;
op.payload->recv_message.recv_message = &recv_stream;
op.payload->recv_message.recv_message_ready = drain_start.get();
s.Op(&op);
s->Op(&op);
f.PushInput(grpc_slice_ref(incoming_data));
});
@ -606,7 +611,7 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
op.payload->recv_initial_metadata.recv_initial_metadata_ready =
do_nothing.get();
op.on_complete = c.get();
s.Op(&op);
s->Op(&op);
f.PushInput(SLICE_FROM_BUFFER(
"\x00\x00\x00\x04\x00\x00\x00\x00\x00"
// Generated using:
@ -624,12 +629,12 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
reset_op();
op.cancel_stream = true;
op.payload->cancel_stream.cancel_error = GRPC_ERROR_CANCELLED;
s.Op(&op);
s.DestroyThen(MakeOnceClosure([](grpc_error* error) {}));
f.FlushExecCtx();
track_counters.Finish(state);
s->Op(&op);
s->DestroyThen(MakeOnceClosure([s](grpc_error* error) { delete s; }));
grpc_metadata_batch_destroy(&b);
grpc_metadata_batch_destroy(&b_recv);
f.FlushExecCtx();
track_counters.Finish(state);
grpc_slice_unref(incoming_data);
}
BENCHMARK(BM_TransportStreamRecv)->Range(0, 128 * 1024 * 1024);

Loading…
Cancel
Save