Merge branch 'master' of github.com:google/grpc into docker_python

pull/733/head
Donna Dionne 10 years ago
commit dc40babc5c
  1. 53
      CONTRIBUTING.md
  2. 79
      Makefile
  3. 2
      README.md
  4. 28
      build.json
  5. 5114
      etc/roots.pem
  6. 7
      include/grpc/grpc_security.h
  7. 11
      src/core/security/security_context.c
  8. 2
      src/node/examples/route_guide.proto
  9. 90
      src/python/src/grpc/_adapter/fore.py
  10. 75
      src/python/src/grpc/_adapter/rear.py
  11. 2
      src/ruby/bin/interop/interop_client.rb
  12. 7
      templates/Makefile.template
  13. 3
      test/cpp/interop/interop_test.cc
  14. 11
      tools/distpackages/build_deb_packages.sh
  15. 13
      tools/dockerfile/grpc_build_deb/Dockerfile
  16. 31
      tools/dockerfile/grpc_ruby_deb/Dockerfile
  17. 5
      tools/dockerfile/grpc_ruby_deb/README.md
  18. 8
      tools/run_tests/tests.json
  19. 12
      vsprojects/vs2013/Grpc.mak

@ -0,0 +1,53 @@
# How to contribute
We definitely welcome patches and contribution to grpc! Here is some guideline
and information about how to do so.
## Getting started
### Legal requirements
In order to protect both you and ourselves, you will need to sign the
[Contributor License Agreement](https://cla.developers.google.com/clas).
### Technical requirements
You will need several tools to work with this repository. In addition to all of
the packages described in the [INSTALL](INSTALL) file, you will also need
python, and the mako template renderer. To install the latter, using pip, one
should simply be able to do `pip install mako`.
In order to run all of the tests we provide, you will need valgrind and clang.
More specifically, under debian, you will need the package libc++-dev to
properly run all the tests.
If you are planning to work on any of the languages other than C and C++, you
will also need their appropriate development environments.
If you want to work under Windows, we recommend you to use Visual Studio 2013.
The [Community or Express editions](http://www.visualstudio.com/en-us/downloads/download-visual-studio-vs.aspx)
are free and suitable for developing with grpc. Note however that our test
environment and tools are available for Unix environments only at the moment.
## Testing your changes
We provide a tool to help you run our suite of tests in various environments.
In order to run most of the available tests, one would need to run:
`./tools/run_tests/run_tests.py`
If you want to run all the possible tests for all possible languages, do this:
`./tools/run_tests/run_tests.py -lall -call`
## Adding or removing source code
Each language uses its own build system to work. Currently, the root's Makefile
and the Visual Studio project files are building the C and C++ source code only
at the moment. In order to ease the maintenance of these files, we have a
template system. Please do not contribute manual changes to any of the generated
files. Instead, modify the template files, or the build.json file, and
re-generate the project files using the following command:
`./tools/buildgen/generate_projects.sh`

File diff suppressed because one or more lines are too long

@ -92,7 +92,7 @@ messages are delivered in the order they were sent.
#Protocol
The gRPC protocol specifies the abstract requirements for communication between
The [gRPC protocol](https://github.com/grpc/grpc-common/blob/master/PROTOCOL-HTTP2.md) specifies the abstract requirements for communication between
clients and servers. A concrete embedding over HTTP/2 completes the picture by
fleshing out the details of each of the required operations.

@ -1271,20 +1271,6 @@
"gpr"
]
},
{
"name": "interop_test",
"build": "test",
"language": "c",
"src": [
"test/cpp/interop/interop_test.c"
],
"deps": [
"grpc_test_util",
"grpc",
"gpr_test_util",
"gpr"
]
},
{
"name": "json_rewrite",
"build": "test",
@ -1714,6 +1700,20 @@
"gpr"
]
},
{
"name": "interop_test",
"build": "test",
"language": "c++",
"src": [
"test/cpp/interop/interop_test.cc"
],
"deps": [
"grpc_test_util",
"grpc",
"gpr_test_util",
"gpr"
]
},
{
"name": "pubsub_client",
"build": "test",

File diff suppressed because it is too large Load Diff

@ -73,8 +73,11 @@ typedef struct {
/* Creates an SSL credentials object.
- pem_roots_cert is the NULL-terminated string containing the PEM encoding
of the server root certificates. If this parameter is NULL, the default
roots will be used.
of the server root certificates. If this parameter is NULL, the
implementation will first try to dereference the file pointed by the
GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment variable, and if that fails,
get the roots from a well-known place on disk (in the grpc install
directory).
- pem_key_cert_pair is a pointer on the object containing client's private
key and certificate chain. This parameter can be NULL if the client does
not have such a key/cert pair. */

@ -61,9 +61,9 @@
"SHA256:AES256-SHA256"
#ifndef INSTALL_PREFIX
static const char *installed_roots_path = "/etc/grpc/roots.pem";
static const char *installed_roots_path = "/usr/share/grpc/roots.pem";
#else
static const char *installed_roots_path = INSTALL_PREFIX "/etc/grpc/roots.pem";
static const char *installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem";
#endif
/* -- Common methods. -- */
@ -404,6 +404,7 @@ static grpc_security_context_vtable ssl_server_vtable = {
static gpr_slice default_pem_root_certs;
static void init_default_pem_root_certs(void) {
/* First try to load the roots from the environment. */
char *default_root_certs_path =
gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
if (default_root_certs_path == NULL) {
@ -412,7 +413,11 @@ static void init_default_pem_root_certs(void) {
default_pem_root_certs = gpr_load_file(default_root_certs_path, NULL);
gpr_free(default_root_certs_path);
}
(void) installed_roots_path;
/* Fall back to installed certs if needed. */
if (GPR_SLICE_IS_EMPTY(default_pem_root_certs)) {
default_pem_root_certs = gpr_load_file(installed_roots_path, NULL);
}
}
size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) {

@ -29,7 +29,7 @@
syntax = "proto3";
option java_package = "ex.grpc";
option java_package = "io.grpc.examples";
package examples;

@ -41,6 +41,9 @@ from grpc.framework.base.packets import interfaces as ticket_interfaces
from grpc.framework.base.packets import null
from grpc.framework.base.packets import packets as tickets
from grpc.framework.foundation import activated
from grpc.framework.foundation import logging_pool
_THREAD_POOL_SIZE = 100
@enum.unique
@ -353,3 +356,90 @@ class ForeLink(ticket_interfaces.ForeLink, activated.Activated):
self._complete(ticket.operation_id, ticket.payload)
else:
self._cancel(ticket.operation_id)
class _ActivatedForeLink(ticket_interfaces.ForeLink, activated.Activated):
def __init__(
self, port, request_deserializers, response_serializers,
root_certificates, key_chain_pairs):
self._port = port
self._request_deserializers = request_deserializers
self._response_serializers = response_serializers
self._root_certificates = root_certificates
self._key_chain_pairs = key_chain_pairs
self._lock = threading.Lock()
self._pool = None
self._fore_link = None
self._rear_link = null.NULL_REAR_LINK
def join_rear_link(self, rear_link):
with self._lock:
self._rear_link = null.NULL_REAR_LINK if rear_link is None else rear_link
if self._fore_link is not None:
self._fore_link.join_rear_link(rear_link)
def _start(self):
with self._lock:
self._pool = logging_pool.pool(_THREAD_POOL_SIZE)
self._fore_link = ForeLink(
self._pool, self._request_deserializers, self._response_serializers,
self._root_certificates, self._key_chain_pairs, port=self._port)
self._fore_link.join_rear_link(self._rear_link)
self._fore_link.start()
return self
def _stop(self):
with self._lock:
self._fore_link.stop()
self._fore_link = None
self._pool.shutdown(wait=True)
self._pool = None
def __enter__(self):
return self._start()
def __exit__(self, exc_type, exc_val, exc_tb):
self._stop()
return False
def start(self):
return self._start()
def stop(self):
self._stop()
def port(self):
with self._lock:
return None if self._fore_link is None else self._fore_link.port()
def accept_back_to_front_ticket(self, ticket):
with self._lock:
if self._fore_link is not None:
self._fore_link.accept_back_to_front_ticket(ticket)
def activated_fore_link(
port, request_deserializers, response_serializers, root_certificates,
key_chain_pairs):
"""Creates a ForeLink that is also an activated.Activated.
The returned object is only valid for use between calls to its start and stop
methods (or in context when used as a context manager).
Args:
port: The port on which to serve RPCs, or None for a port to be
automatically selected.
request_deserializers: A dictionary from RPC method names to request object
deserializer behaviors.
response_serializers: A dictionary from RPC method names to response object
serializer behaviors.
root_certificates: The PEM-encoded client root certificates as a bytestring
or None.
key_chain_pairs: A sequence of PEM-encoded private key-certificate chain
pairs.
"""
return _ActivatedForeLink(
port, request_deserializers, response_serializers, root_certificates,
key_chain_pairs)

@ -40,6 +40,9 @@ from grpc.framework.base.packets import interfaces as ticket_interfaces
from grpc.framework.base.packets import null
from grpc.framework.base.packets import packets as tickets
from grpc.framework.foundation import activated
from grpc.framework.foundation import logging_pool
_THREAD_POOL_SIZE = 100
_INVOCATION_EVENT_KINDS = (
_low.Event.Kind.METADATA_ACCEPTED,
@ -361,3 +364,75 @@ class RearLink(ticket_interfaces.RearLink, activated.Activated):
else:
# NOTE(nathaniel): All other categories are treated as cancellation.
self._cancel(ticket.operation_id)
class _ActivatedRearLink(ticket_interfaces.RearLink, activated.Activated):
def __init__(self, host, port, request_serializers, response_deserializers):
self._host = host
self._port = port
self._request_serializers = request_serializers
self._response_deserializers = response_deserializers
self._lock = threading.Lock()
self._pool = None
self._rear_link = None
self._fore_link = null.NULL_FORE_LINK
def join_fore_link(self, fore_link):
with self._lock:
self._fore_link = null.NULL_FORE_LINK if fore_link is None else fore_link
def _start(self):
with self._lock:
self._pool = logging_pool.pool(_THREAD_POOL_SIZE)
self._rear_link = RearLink(
self._host, self._port, self._pool, self._request_serializers,
self._response_deserializers)
self._rear_link.join_fore_link(self._fore_link)
self._rear_link.start()
return self
def _stop(self):
with self._lock:
self._rear_link.stop()
self._rear_link = None
self._pool.shutdown(wait=True)
self._pool = None
def __enter__(self):
return self._start()
def __exit__(self, exc_type, exc_val, exc_tb):
self._stop()
return False
def start(self):
return self._start()
def stop(self):
self._stop()
def accept_front_to_back_ticket(self, ticket):
with self._lock:
if self._rear_link is not None:
self._rear_link.accept_front_to_back_ticket(ticket)
def activated_rear_link(
host, port, request_serializers, response_deserializers):
"""Creates a RearLink that is also an activated.Activated.
The returned object is only valid for use between calls to its start and stop
methods (or in context when used as a context manager).
Args:
host: The host to which to connect for RPC service.
port: The port to which to connect for RPC service.
request_serializers: A dictionary from RPC method name to request object
serializer behavior.
response_deserializers: A dictionary from RPC method name to response
object deserializer behavior.
"""
return _ActivatedRearLink(
host, port, request_serializers, response_deserializers)

@ -56,7 +56,7 @@ require 'test/cpp/interop/empty'
require 'signet/ssl_config'
include Google::RPC::Auth
include GRPC::Auth
# loads the certificates used to access the test server securely.
def load_test_certs

@ -729,7 +729,7 @@ $(OBJDIR)/$(CONFIG)/%.o : %.cc
$(Q) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
install: install_c install_cxx install-plugins verify-install
install: install_c install_cxx install-plugins install-certs verify-install
install_c: install-headers_c install-static_c install-shared_c
@ -824,6 +824,11 @@ else
% endfor
endif
install-certs: etc/roots.pem
$(E) "[INSTALL] Installing root certificates"
$(Q) $(INSTALL) -d $(prefix)/share/grpc
$(Q) $(INSTALL) etc/roots.pem $(prefix)/share/grpc/roots.pem
verify-install:
ifeq ($(SYSTEM_OK),true)
@echo "Your system looks ready to go."

@ -44,8 +44,11 @@
#include <sys/types.h>
#include <sys/wait.h>
extern "C" {
#include "src/core/iomgr/socket_utils_posix.h"
#include "src/core/support/string.h"
}
#include <grpc/support/alloc.h>
#include <grpc/support/host_port.h>
#include <grpc/support/log.h>

@ -35,7 +35,13 @@ mkdir -p $deb_dest
version='0.8.0.0'
arch=`uname -p`
if [ -f /.dockerinit ]; then
# We're in Docker where uname -p returns "unknown".
arch=x86_64
else
arch=`uname -p`
fi
if [ $arch != "x86_64" ]
then
echo Unsupported architecture.
@ -77,13 +83,14 @@ do
# create symlinks to shared libraries
for libname in $arch_lib_dir/*.a
do
base=`basename -s .a $libname`
base=`basename $libname .a`
ln -s $base.so.$version $arch_lib_dir/$base.so
done
fi
# Adjust mode for some files in the package
find $tmp_dir/$pkg_name -type d | xargs chmod 755
find $tmp_dir/$pkg_name -type d | xargs chmod a-s
find $tmp_dir/$pkg_name -type f | xargs chmod 644
chmod 755 $tmp_dir/$pkg_name/DEBIAN/{postinst,postrm}

@ -0,0 +1,13 @@
# Dockerfile to build Debian packages for gRPC C core.
FROM grpc/base
# Install dependencies
RUN apt-get update && apt-get install -y lintian
# Get the source from GitHub
RUN git clone git@github.com:grpc/grpc.git /var/local/git/grpc
RUN cd /var/local/git/grpc && \
git pull --recurse-submodules && \
git submodule update --init --recursive
RUN /bin/bash -l -c 'cd /var/local/git/grpc/tools/distpackages && ./build_deb_packages.sh'

@ -0,0 +1,31 @@
# Dockerfile for gRPC Ruby, but using Debian packages for gRPC C core.
FROM grpc/ruby_base
# Pull the latest sources
RUN cd /var/local/git/grpc \
&& git pull --recurse-submodules \
&& git submodule update --init --recursive
# Make sure we don't rely on things that shouldn't be there.
RUN make clean -C /var/local/git/grpc
# Debian packages need to be supplied externally
ADD libgrpc_amd64.deb libgrpc_amd64.deb
ADD libgrpc-dev_amd64.deb libgrpc-dev_amd64.deb
# Install the C core .deb packages
RUN /bin/bash -l -c 'dpkg -i libgrpc_amd64.deb libgrpc-dev_amd64.deb'
# Build ruby gRPC and run its tests
RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake'
# Add a cacerts directory containing the Google root pem file, allowing the
# ruby client to access the production test instance
ADD cacerts cacerts
# Add a service_account directory containing the auth creds file
ADD service_account service_account
# Specify the default command such that the interop server runs on its known
# testing port
CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --use_tls --port 8060"]

@ -0,0 +1,5 @@
GRPC RUBY Base Dockerfile (Debian package version)
========================
Dockerfile for creating the Ruby gRPC development Docker instance.
Uses gRPC C core Debian packages instead of installing it using make.

@ -189,10 +189,6 @@
"language": "c",
"name": "httpcli_parser_test"
},
{
"language": "c",
"name": "interop_test"
},
{
"language": "c",
"name": "json_rewrite_test"
@ -285,6 +281,10 @@
"language": "c++",
"name": "end2end_test"
},
{
"language": "c++",
"name": "interop_test"
},
{
"language": "c++",
"name": "pubsub_publisher_test"

@ -25,10 +25,10 @@ grpc_test_util:
$(OUT_DIR):
mkdir $(OUT_DIR)
buildtests: alarm_heap_test.exe alarm_list_test.exe alarm_test.exe alpn_test.exe bin_encoder_test.exe census_hash_table_test.exe census_statistics_multiple_writers_circular_buffer_test.exe census_statistics_multiple_writers_test.exe census_statistics_performance_test.exe census_statistics_quick_test.exe census_statistics_small_log_test.exe census_stats_store_test.exe census_stub_test.exe census_trace_store_test.exe census_window_stats_test.exe chttp2_status_conversion_test.exe chttp2_stream_encoder_test.exe chttp2_stream_map_test.exe chttp2_transport_end2end_test.exe dualstack_socket_test.exe echo_test.exe fd_posix_test.exe fling_stream_test.exe fling_test.exe gpr_cancellable_test.exe gpr_cmdline_test.exe gpr_env_test.exe gpr_file_test.exe gpr_histogram_test.exe gpr_host_port_test.exe gpr_log_test.exe gpr_slice_buffer_test.exe gpr_slice_test.exe gpr_string_test.exe gpr_sync_test.exe gpr_thd_test.exe gpr_time_test.exe gpr_useful_test.exe grpc_base64_test.exe grpc_byte_buffer_reader_test.exe grpc_channel_stack_test.exe grpc_completion_queue_test.exe grpc_credentials_test.exe grpc_json_token_test.exe grpc_stream_op_test.exe hpack_parser_test.exe hpack_table_test.exe httpcli_format_request_test.exe httpcli_parser_test.exe httpcli_test.exe interop_test.exe json_rewrite_test.exe json_test.exe lame_client_test.exe message_compress_test.exe metadata_buffer_test.exe multi_init_test.exe murmur_hash_test.exe no_server_test.exe poll_kick_posix_test.exe resolve_address_test.exe secure_endpoint_test.exe sockaddr_utils_test.exe tcp_client_posix_test.exe tcp_posix_test.exe tcp_server_posix_test.exe time_averaged_stats_test.exe time_test.exe timeout_encoding_test.exe transport_metadata_test.exe
buildtests: alarm_heap_test.exe alarm_list_test.exe alarm_test.exe alpn_test.exe bin_encoder_test.exe census_hash_table_test.exe census_statistics_multiple_writers_circular_buffer_test.exe census_statistics_multiple_writers_test.exe census_statistics_performance_test.exe census_statistics_quick_test.exe census_statistics_small_log_test.exe census_stats_store_test.exe census_stub_test.exe census_trace_store_test.exe census_window_stats_test.exe chttp2_status_conversion_test.exe chttp2_stream_encoder_test.exe chttp2_stream_map_test.exe chttp2_transport_end2end_test.exe dualstack_socket_test.exe echo_test.exe fd_posix_test.exe fling_stream_test.exe fling_test.exe gpr_cancellable_test.exe gpr_cmdline_test.exe gpr_env_test.exe gpr_file_test.exe gpr_histogram_test.exe gpr_host_port_test.exe gpr_log_test.exe gpr_slice_buffer_test.exe gpr_slice_test.exe gpr_string_test.exe gpr_sync_test.exe gpr_thd_test.exe gpr_time_test.exe gpr_useful_test.exe grpc_base64_test.exe grpc_byte_buffer_reader_test.exe grpc_channel_stack_test.exe grpc_completion_queue_test.exe grpc_credentials_test.exe grpc_json_token_test.exe grpc_stream_op_test.exe hpack_parser_test.exe hpack_table_test.exe httpcli_format_request_test.exe httpcli_parser_test.exe httpcli_test.exe json_rewrite_test.exe json_test.exe lame_client_test.exe message_compress_test.exe metadata_buffer_test.exe multi_init_test.exe murmur_hash_test.exe no_server_test.exe poll_kick_posix_test.exe resolve_address_test.exe secure_endpoint_test.exe sockaddr_utils_test.exe tcp_client_posix_test.exe tcp_posix_test.exe tcp_server_posix_test.exe time_averaged_stats_test.exe time_test.exe timeout_encoding_test.exe transport_metadata_test.exe
echo All tests built.
test: alarm_heap_test alarm_list_test alarm_test alpn_test bin_encoder_test census_hash_table_test census_statistics_multiple_writers_circular_buffer_test census_statistics_multiple_writers_test census_statistics_performance_test census_statistics_quick_test census_statistics_small_log_test census_stats_store_test census_stub_test census_trace_store_test census_window_stats_test chttp2_status_conversion_test chttp2_stream_encoder_test chttp2_stream_map_test chttp2_transport_end2end_test dualstack_socket_test echo_test fd_posix_test fling_stream_test fling_test gpr_cancellable_test gpr_cmdline_test gpr_env_test gpr_file_test gpr_histogram_test gpr_host_port_test gpr_log_test gpr_slice_buffer_test gpr_slice_test gpr_string_test gpr_sync_test gpr_thd_test gpr_time_test gpr_useful_test grpc_base64_test grpc_byte_buffer_reader_test grpc_channel_stack_test grpc_completion_queue_test grpc_credentials_test grpc_json_token_test grpc_stream_op_test hpack_parser_test hpack_table_test httpcli_format_request_test httpcli_parser_test httpcli_test interop_test json_rewrite_test json_test lame_client_test message_compress_test metadata_buffer_test multi_init_test murmur_hash_test no_server_test poll_kick_posix_test resolve_address_test secure_endpoint_test sockaddr_utils_test tcp_client_posix_test tcp_posix_test tcp_server_posix_test time_averaged_stats_test time_test timeout_encoding_test transport_metadata_test
test: alarm_heap_test alarm_list_test alarm_test alpn_test bin_encoder_test census_hash_table_test census_statistics_multiple_writers_circular_buffer_test census_statistics_multiple_writers_test census_statistics_performance_test census_statistics_quick_test census_statistics_small_log_test census_stats_store_test census_stub_test census_trace_store_test census_window_stats_test chttp2_status_conversion_test chttp2_stream_encoder_test chttp2_stream_map_test chttp2_transport_end2end_test dualstack_socket_test echo_test fd_posix_test fling_stream_test fling_test gpr_cancellable_test gpr_cmdline_test gpr_env_test gpr_file_test gpr_histogram_test gpr_host_port_test gpr_log_test gpr_slice_buffer_test gpr_slice_test gpr_string_test gpr_sync_test gpr_thd_test gpr_time_test gpr_useful_test grpc_base64_test grpc_byte_buffer_reader_test grpc_channel_stack_test grpc_completion_queue_test grpc_credentials_test grpc_json_token_test grpc_stream_op_test hpack_parser_test hpack_table_test httpcli_format_request_test httpcli_parser_test httpcli_test json_rewrite_test json_test lame_client_test message_compress_test metadata_buffer_test multi_init_test murmur_hash_test no_server_test poll_kick_posix_test resolve_address_test secure_endpoint_test sockaddr_utils_test tcp_client_posix_test tcp_posix_test tcp_server_posix_test time_averaged_stats_test time_test timeout_encoding_test transport_metadata_test
echo All tests ran.
test_gpr: gpr_cancellable_test gpr_cmdline_test gpr_env_test gpr_file_test gpr_histogram_test gpr_host_port_test gpr_log_test gpr_slice_buffer_test gpr_slice_test gpr_string_test gpr_sync_test gpr_thd_test gpr_time_test gpr_useful_test
@ -498,14 +498,6 @@ httpcli_test: httpcli_test.exe
echo Running httpcli_test
$(OUT_DIR)\httpcli_test.exe
interop_test.exe: grpc_test_util
echo Building interop_test
$(CC) $(CFLAGS) /Fo:$(OUT_DIR)\ ..\..\test\cpp\interop\interop_test.c
$(LINK) $(LFLAGS) /OUT:"$(OUT_DIR)\interop_test.exe" Debug\grpc_test_util.lib Debug\grpc.lib Debug\gpr_test_util.lib Debug\gpr.lib $(LIBS) $(OUT_DIR)\interop_test.obj
interop_test: interop_test.exe
echo Running interop_test
$(OUT_DIR)\interop_test.exe
json_rewrite.exe: grpc_test_util
echo Building json_rewrite
$(CC) $(CFLAGS) /Fo:$(OUT_DIR)\ ..\..\test\core\json\json_rewrite.c

Loading…
Cancel
Save