mirror of https://github.com/grpc/grpc.git
Merge remote-tracking branch 'juanlishen/pf_check_state_before_watch' into pf_keep_unselected_subchannels
Mark D. Roth
6 years ago
commit
e9e92d443c
152 changed files with 2851 additions and 763 deletions
@ -1,4 +1,4 @@ |
||||
set noparent |
||||
@jtattermusch |
||||
@nicolasnoble |
||||
@matt-kwong |
||||
@mehrdada |
||||
|
||||
@ -0,0 +1,36 @@ |
||||
# SSL in gRPC and performance |
||||
|
||||
The SSL requirement of gRPC isn't necessarily making it easy to integrate. The HTTP/2 protocol requires ALPN support, which is a fairly new handshake protocol only supported by recent implementations. |
||||
|
||||
As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code |
||||
(by setting the `OPENSSL_NO_ASM` option), which can impact performance by an order of magnitude when processing encrypted streams. |
||||
|
||||
## gRPC C++: Building from Source |
||||
|
||||
Build system | Condition | Platform | Uses assembly optimizations |
||||
---|---|---|-- |
||||
Makefile | with OpenSSL 1.0.2 development files | all | :heavy_check_mark: |
||||
Makefile | all other cases | all | :x: |
||||
Bazel | | Linux | :heavy_check_mark: |
||||
Bazel | | MacOS | :heavy_check_mark: |
||||
Bazel | | Windows | :x: |
||||
CMake | boringssl from submodule (default) | all | :x: |
||||
CMake | pre-installed OpenSSL 1.0.2+ (`gRPC_SSL_PROVIDER=package`) | all | :heavy_check_mark: |
||||
|
||||
## Other Languages: Binary/Source Packages |
||||
|
||||
In addition, we are shipping packages for language implementations. These packages are source packages, but also have pre-built binaries being distributed. Building packages from source may give a different result in some cases. |
||||
|
||||
Language | From source | Platform | Uses assembly optimizations |
||||
---|---|---|--- |
||||
C# | n/a | all | :x: |
||||
Node.JS | n/a | Linux | :heavy_check_mark: |
||||
Node.JS | n/a | MacOS | :heavy_check_mark: |
||||
Node.JS | n/a | Windows | :x: |
||||
Electron | n/a | all | :heavy_check_mark: |
||||
ObjC | Yes | iOS | :x: |
||||
PHP | Yes | all | Same as the `Makefile` case from above |
||||
PHP | No | all | :x: |
||||
Python | n/a | all | :x: |
||||
Ruby | No | all | :x: |
||||
|
||||
@ -0,0 +1,75 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2015 gRPC authors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include "src/core/lib/iomgr/port.h" |
||||
|
||||
#ifdef GRPC_CFSTREAM_IOMGR |
||||
|
||||
#include "src/core/lib/debug/trace.h" |
||||
#include "src/core/lib/iomgr/ev_posix.h" |
||||
#include "src/core/lib/iomgr/iomgr_internal.h" |
||||
#include "src/core/lib/iomgr/iomgr_posix.h" |
||||
#include "src/core/lib/iomgr/resolve_address.h" |
||||
#include "src/core/lib/iomgr/tcp_client.h" |
||||
#include "src/core/lib/iomgr/tcp_posix.h" |
||||
#include "src/core/lib/iomgr/tcp_server.h" |
||||
#include "src/core/lib/iomgr/timer.h" |
||||
|
||||
static const char* grpc_cfstream_env_var = "grpc_cfstream"; |
||||
|
||||
extern grpc_tcp_server_vtable grpc_posix_tcp_server_vtable; |
||||
extern grpc_tcp_client_vtable grpc_posix_tcp_client_vtable; |
||||
extern grpc_tcp_client_vtable grpc_cfstream_client_vtable; |
||||
extern grpc_timer_vtable grpc_generic_timer_vtable; |
||||
extern grpc_pollset_vtable grpc_posix_pollset_vtable; |
||||
extern grpc_pollset_set_vtable grpc_posix_pollset_set_vtable; |
||||
extern grpc_address_resolver_vtable grpc_posix_resolver_vtable; |
||||
|
||||
static void iomgr_platform_init(void) { |
||||
grpc_wakeup_fd_global_init(); |
||||
grpc_event_engine_init(); |
||||
} |
||||
|
||||
static void iomgr_platform_flush(void) {} |
||||
|
||||
static void iomgr_platform_shutdown(void) { |
||||
grpc_event_engine_shutdown(); |
||||
grpc_wakeup_fd_global_destroy(); |
||||
} |
||||
|
||||
static grpc_iomgr_platform_vtable vtable = { |
||||
iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown}; |
||||
|
||||
void grpc_set_default_iomgr_platform() { |
||||
char* enable_cfstream = getenv(grpc_cfstream_env_var); |
||||
grpc_tcp_client_vtable* client_vtable = &grpc_posix_tcp_client_vtable; |
||||
if (enable_cfstream != nullptr && enable_cfstream[0] == '1') { |
||||
client_vtable = &grpc_cfstream_client_vtable; |
||||
} |
||||
grpc_set_tcp_client_impl(client_vtable); |
||||
grpc_set_tcp_server_impl(&grpc_posix_tcp_server_vtable); |
||||
grpc_set_timer_impl(&grpc_generic_timer_vtable); |
||||
grpc_set_pollset_vtable(&grpc_posix_pollset_vtable); |
||||
grpc_set_pollset_set_vtable(&grpc_posix_pollset_set_vtable); |
||||
grpc_set_resolver_impl(&grpc_posix_resolver_vtable); |
||||
grpc_set_iomgr_platform_vtable(&vtable); |
||||
} |
||||
|
||||
#endif /* GRPC_CFSTREAM_IOMGR */ |
||||
@ -0,0 +1,29 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2018 gRPC authors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H |
||||
#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H |
||||
|
||||
namespace grpc_core { |
||||
|
||||
// Returns a slice containing roots from the OS trust store
|
||||
grpc_slice LoadSystemRootCerts(); |
||||
|
||||
} // namespace grpc_core
|
||||
|
||||
#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ |
||||
@ -0,0 +1,32 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2018 gRPC authors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include <grpc/slice_buffer.h> |
||||
#include "src/core/lib/security/security_connector/load_system_roots.h" |
||||
|
||||
#ifndef GPR_LINUX |
||||
|
||||
namespace grpc_core { |
||||
|
||||
grpc_slice LoadSystemRootCerts() { return grpc_empty_slice(); } |
||||
|
||||
} // namespace grpc_core
|
||||
|
||||
#endif /* GPR_LINUX */ |
||||
@ -0,0 +1,165 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2018 gRPC authors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include <grpc/slice_buffer.h> |
||||
#include "src/core/lib/security/security_connector/load_system_roots_linux.h" |
||||
|
||||
#ifdef GPR_LINUX |
||||
|
||||
#include "src/core/lib/security/security_connector/load_system_roots.h" |
||||
|
||||
#include <dirent.h> |
||||
#include <fcntl.h> |
||||
#include <stdbool.h> |
||||
#include <string.h> |
||||
#include <sys/param.h> |
||||
#include <sys/stat.h> |
||||
#include <sys/types.h> |
||||
#include <unistd.h> |
||||
|
||||
#include <grpc/support/alloc.h> |
||||
#include <grpc/support/log.h> |
||||
#include <grpc/support/string_util.h> |
||||
|
||||
#include "src/core/lib/gpr/env.h" |
||||
#include "src/core/lib/gpr/string.h" |
||||
#include "src/core/lib/gpr/useful.h" |
||||
#include "src/core/lib/gprpp/inlined_vector.h" |
||||
#include "src/core/lib/iomgr/load_file.h" |
||||
|
||||
namespace grpc_core { |
||||
namespace { |
||||
|
||||
const char* kLinuxCertFiles[] = { |
||||
"/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", |
||||
"/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem", |
||||
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}; |
||||
const char* kLinuxCertDirectories[] = { |
||||
"/etc/ssl/certs", "/system/etc/security/cacerts", "/usr/local/share/certs", |
||||
"/etc/pki/tls/certs", "/etc/openssl/certs"}; |
||||
|
||||
grpc_slice GetSystemRootCerts() { |
||||
grpc_slice valid_bundle_slice = grpc_empty_slice(); |
||||
size_t num_cert_files_ = GPR_ARRAY_SIZE(kLinuxCertFiles); |
||||
for (size_t i = 0; i < num_cert_files_; i++) { |
||||
grpc_error* error = |
||||
grpc_load_file(kLinuxCertFiles[i], 1, &valid_bundle_slice); |
||||
if (error == GRPC_ERROR_NONE) { |
||||
return valid_bundle_slice; |
||||
} |
||||
} |
||||
return grpc_empty_slice(); |
||||
} |
||||
|
||||
} // namespace
|
||||
|
||||
void GetAbsoluteFilePath(const char* valid_file_dir, |
||||
const char* file_entry_name, char* path_buffer) { |
||||
if (valid_file_dir != nullptr && file_entry_name != nullptr) { |
||||
int path_len = snprintf(path_buffer, MAXPATHLEN, "%s/%s", valid_file_dir, |
||||
file_entry_name); |
||||
if (path_len == 0) { |
||||
gpr_log(GPR_ERROR, "failed to get absolute path for file: %s", |
||||
file_entry_name); |
||||
} |
||||
} |
||||
} |
||||
|
||||
grpc_slice CreateRootCertsBundle(const char* certs_directory) { |
||||
grpc_slice bundle_slice = grpc_empty_slice(); |
||||
if (certs_directory == nullptr) { |
||||
return bundle_slice; |
||||
} |
||||
DIR* ca_directory = opendir(certs_directory); |
||||
if (ca_directory == nullptr) { |
||||
return bundle_slice; |
||||
} |
||||
struct FileData { |
||||
char path[MAXPATHLEN]; |
||||
off_t size; |
||||
}; |
||||
InlinedVector<FileData, 2> roots_filenames; |
||||
size_t total_bundle_size = 0; |
||||
struct dirent* directory_entry; |
||||
while ((directory_entry = readdir(ca_directory)) != nullptr) { |
||||
struct stat dir_entry_stat; |
||||
const char* file_entry_name = directory_entry->d_name; |
||||
FileData file_data; |
||||
GetAbsoluteFilePath(certs_directory, file_entry_name, file_data.path); |
||||
int stat_return = stat(file_data.path, &dir_entry_stat); |
||||
if (stat_return == -1 || !S_ISREG(dir_entry_stat.st_mode)) { |
||||
// no subdirectories.
|
||||
if (stat_return == -1) { |
||||
gpr_log(GPR_ERROR, "failed to get status for file: %s", file_data.path); |
||||
} |
||||
continue; |
||||
} |
||||
file_data.size = dir_entry_stat.st_size; |
||||
total_bundle_size += file_data.size; |
||||
roots_filenames.push_back(file_data); |
||||
} |
||||
closedir(ca_directory); |
||||
char* bundle_string = static_cast<char*>(gpr_zalloc(total_bundle_size + 1)); |
||||
size_t bytes_read = 0; |
||||
for (size_t i = 0; i < roots_filenames.size(); i++) { |
||||
int file_descriptor = open(roots_filenames[i].path, O_RDONLY); |
||||
if (file_descriptor != -1) { |
||||
// Read file into bundle.
|
||||
size_t cert_file_size = roots_filenames[i].size; |
||||
int read_ret = |
||||
read(file_descriptor, bundle_string + bytes_read, cert_file_size); |
||||
if (read_ret != -1) { |
||||
bytes_read += read_ret; |
||||
} else { |
||||
gpr_log(GPR_ERROR, "failed to read file: %s", roots_filenames[i].path); |
||||
} |
||||
} |
||||
} |
||||
bundle_slice = grpc_slice_new(bundle_string, bytes_read, gpr_free); |
||||
return bundle_slice; |
||||
} |
||||
|
||||
grpc_slice LoadSystemRootCerts() { |
||||
grpc_slice result = grpc_empty_slice(); |
||||
// Prioritize user-specified custom directory if flag is set.
|
||||
char* custom_dir = gpr_getenv("GRPC_SYSTEM_SSL_ROOTS_DIR"); |
||||
if (custom_dir != nullptr) { |
||||
result = CreateRootCertsBundle(custom_dir); |
||||
gpr_free(custom_dir); |
||||
} |
||||
// If the custom directory is empty/invalid/not specified, fallback to
|
||||
// distribution-specific directory.
|
||||
if (GRPC_SLICE_IS_EMPTY(result)) { |
||||
result = GetSystemRootCerts(); |
||||
} |
||||
if (GRPC_SLICE_IS_EMPTY(result)) { |
||||
for (size_t i = 0; i < GPR_ARRAY_SIZE(kLinuxCertDirectories); i++) { |
||||
result = CreateRootCertsBundle(kLinuxCertDirectories[i]); |
||||
if (!GRPC_SLICE_IS_EMPTY(result)) { |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
} // namespace grpc_core
|
||||
|
||||
#endif /* GPR_LINUX */ |
||||
@ -0,0 +1,44 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2018 gRPC authors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H |
||||
#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#ifdef GPR_LINUX |
||||
|
||||
namespace grpc_core { |
||||
|
||||
// Creates a bundle slice containing the contents of all certificate files in
|
||||
// a directory.
|
||||
// Returns such slice.
|
||||
// Exposed for testing purposes only.
|
||||
grpc_slice CreateRootCertsBundle(const char* certs_directory); |
||||
|
||||
// Gets the absolute file path needed to load a certificate file.
|
||||
// Populates path_buffer, which must be of size MAXPATHLEN.
|
||||
// Exposed for testing purposes only.
|
||||
void GetAbsoluteFilePath(const char* valid_file_dir, |
||||
const char* file_entry_name, char* path_buffer); |
||||
|
||||
} // namespace grpc_core
|
||||
|
||||
#endif /* GPR_LINUX */ |
||||
#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H \ |
||||
*/ |
||||
@ -1,25 +1,22 @@ |
||||
<?xml version="1.0" encoding="utf-8"?> |
||||
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> |
||||
<PropertyGroup> |
||||
<_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\</_GrpcCoreNugetNativePath> |
||||
</PropertyGroup> |
||||
|
||||
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'"> |
||||
<AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\arm64-v8a\libgrpc_csharp_ext.so"> |
||||
<AndroidNativeLibrary Include="$(MSBuildThisFileDirectory)..\..\native\android\arm64-v8a\libgrpc_csharp_ext.so"> |
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory> |
||||
<Abi>arm64-v8a</Abi> |
||||
</AndroidNativeLibrary> |
||||
</ItemGroup> |
||||
|
||||
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'"> |
||||
<AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\armeabi-v7a\libgrpc_csharp_ext.so"> |
||||
<AndroidNativeLibrary Include="$(MSBuildThisFileDirectory)..\..\native\android\armeabi-v7a\libgrpc_csharp_ext.so"> |
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory> |
||||
<Abi>armeabi-v7a</Abi> |
||||
</AndroidNativeLibrary> |
||||
</ItemGroup> |
||||
|
||||
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'"> |
||||
<AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\x86\libgrpc_csharp_ext.so"> |
||||
<AndroidNativeLibrary Include="$(MSBuildThisFileDirectory)..\..\native\android\x86\libgrpc_csharp_ext.so"> |
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory> |
||||
<Abi>x86</Abi> |
||||
</AndroidNativeLibrary> |
||||
@ -0,0 +1,191 @@ |
||||
# Project overview |
||||
|
||||
## Title |
||||
|
||||
Enable Building of gRPC Python with Bazel |
||||
|
||||
## Overview |
||||
|
||||
gRPC Python currently has a constellation of scripts written to build the |
||||
project, but it has a lot of limitations in terms of speed and maintainability. |
||||
[Bazel](https://bazel.build/) is the open-sourced variant of Google's internal |
||||
system, Blaze, which is an ideal replacement for building such projects in a |
||||
fast and declarative fashion. But Bazel in itself is still in active |
||||
development, especially in terms of Python (amongst a few other languages). |
||||
|
||||
The project aimed to fill this gap and build gRPC Python with Bazel. |
||||
|
||||
[Project page](https://summerofcode.withgoogle.com/projects/#6482576244473856) |
||||
|
||||
[Link to proposal](https://storage.googleapis.com/summerofcode-prod.appspot.com/gsoc/core_project/doc/5316764725411840_1522049732_Naresh_Ramesh_-_GSoC_proposal.pdf) |
||||
|
||||
## Thoughts and challenges |
||||
|
||||
### State of Bazel for Python |
||||
|
||||
Although previously speculated, the project didn't require any contributions |
||||
directly to [bazelbuild/bazel](https://github.com/bazelbuild/bazel). The Bazel |
||||
rules for Python are currently being separated out into their own repo at |
||||
[bazelbuild/rules_python](https://github.com/bazelbuild/rules_python/). |
||||
|
||||
Bazel is [still very much in active development for |
||||
Python](https://groups.google.com/forum/#!topic/bazel-sig-python/iQjV9sfSufw) |
||||
though. There's still challenges when it comes to building for Python 2 vs 3. |
||||
Using pip packages is still in experimental. Bazel Python support is currently |
||||
distributed across these two repositories and is yet to begin migration to one |
||||
place (which will be |
||||
[bazelbuild/rules_python](https://github.com/bazelbuild/rules_python/)). |
||||
|
||||
Bazel's roadmap for Python is publicly available [here as a Google |
||||
doc](https://docs.google.com/document/d/1A6J3j3y1SQ0HliS86_mZBnB5UeBe7vExWL2Ryd_EONI/edit). |
||||
|
||||
### Cross collaboration between projects |
||||
|
||||
Cross contribution surprisingly came up because of building protobuf sources |
||||
for Python, which is still not natively supported by Bazel. An existing |
||||
repository, [pubref/rules_protobuf](https://github.com/pubref/rules_protobuf), |
||||
which was maintained by an independent maintainer (i.e. not a part of Bazel) |
||||
helped solve this problem, but had [one major blocking |
||||
issue](https://github.com/pubref/rules_protobuf/issues/233) and could not be |
||||
resolved at the source. But [a solution to the |
||||
issue](https://github.com/pubref/rules_protobuf/pull/196) was proposed by user |
||||
dududko, which was not merged because of failing golang tests but worked well |
||||
for Python. Hence, a fork of this repo was made and is to be used with gRPC |
||||
until the solution can be merged back at the source. |
||||
|
||||
### Building Cython code |
||||
|
||||
Building Cython code is still not supported by Bazel, but the team at |
||||
[cython/cython](https://github.com/cython/cython) have added support for Bazel |
||||
on their side. The way it works is by including Cython as a third-party Bazel |
||||
dependency and using custom Bazel rules for building our Cython code using the |
||||
binary within the dependency. |
||||
|
||||
### Packaging Python code using Bazel |
||||
|
||||
pip and PyPI still remain the de-facto standard for distributing Python |
||||
packages. Although Bazel is pretty versatile and is amazing for it's |
||||
reproducible and incremental build capabilities, these can only be still used |
||||
by the contributors and developers for building and testing the gRPC code. But |
||||
there's no way yet to build Python packages for distribution. |
||||
|
||||
### Building gRPC Python with Bazel on Kokoro (internal CI) |
||||
|
||||
Integration with the internal CI was one of the areas that highlighted how |
||||
simple Bazel can be to use. gRPC was already using a dockerized Bazel setup to |
||||
build some of it's core code (but not as the primary build setup). Adding a new |
||||
job on the internal CI ended up being as simple as creating a new shell script |
||||
to install the required dependencies (which were python-dev and Bazel) and a |
||||
new configuration file which pointed to the subdirectiory (src/python) under |
||||
which to look for targets and run the tests accordingly. |
||||
|
||||
### Handling imports in Python code |
||||
|
||||
When writing Python packages, imports in nested modules are typically made |
||||
relative to the package root. But because of the way Bazel works, these paths |
||||
wouldn't make sense from the Workspace root. So, the folks at Bazel have added |
||||
a nifty `imports` parameter to all the Python rules which lets us specify for |
||||
each target, which path to consider as the root. This parameter allows for |
||||
relative paths like `imports = ["../",]`. |
||||
|
||||
### Fetching Python headers for Cython code to use |
||||
|
||||
Cython code makes use of `Python.h`, which pulls in the Python API for C |
||||
extension modules to use, but it's location depending on the Python version and |
||||
operating system the code is building on. To make this easier, the folks at |
||||
Tensorflow wrote [repository rules for Python |
||||
autoconfiguration](https://github.com/tensorflow/tensorflow/tree/e447ae4759317156d31a9421290716f0ffbffcd8/third_party/py). |
||||
This has been [adapted with some some |
||||
modifications](https://github.com/grpc/grpc/pull/15992) for use in gRPC Python |
||||
as well. |
||||
|
||||
## How to use |
||||
|
||||
All the Bazel tests for gRPC Python can be run using a single command: |
||||
|
||||
```bash |
||||
bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/... |
||||
``` |
||||
|
||||
If any specific test is to be run, like say `LoggingPoolTest` (which is present |
||||
in |
||||
`src/python/grpcio_tests/tests/unit/framework/foundation/_logging_pool_test.py`), |
||||
the command to run would be: |
||||
|
||||
```bash |
||||
bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/grpcio_tests/tests/unit/framework/foundation:logging_pool_test |
||||
``` |
||||
|
||||
where, `logging_pool_test` is the name of the Bazel target for this test. |
||||
|
||||
Similarly, to run a particular method, use: |
||||
|
||||
```bash |
||||
bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/grpcio_tests/tests/unit/_rpc_test --test_arg=RPCTest.testUnrecognizedMethod |
||||
``` |
||||
|
||||
## Useful Bazel flags |
||||
|
||||
- Use `bazel build` with a `-s` flag to see the logs being printed out to |
||||
standard output while building. |
||||
- Similarly, use `bazel test` with a `--test_output=streamed` to see the the |
||||
test logs while testing. Something to know while using this flag is that all |
||||
tests will be run locally, without sharding, one at a time. |
||||
|
||||
## Contributions |
||||
|
||||
### Related to the project |
||||
|
||||
- [435c6f8](https://github.com/grpc/grpc/commit/435c6f8d1e53783ec049b3482445813afd8bc514) |
||||
Update grpc_gevent cython files to include .pxi |
||||
- [74426fd](https://github.com/grpc/grpc/commit/74426fd2164c51d6754732ebe372133c19ba718c) |
||||
Add gevent_util.h to grpc_base_c Bazel target |
||||
- [b6518af](https://github.com/grpc/grpc/commit/b6518afdd610f0115b42aee1ffc71520c6b0d6b1) |
||||
Upgrade Bazel to 0.15.0 |
||||
- [ebcf04d](https://github.com/grpc/grpc/commit/ebcf04d075333c42979536c5dd2091d363f67e5a) |
||||
Kokoro setup for building gRPC Python with Bazel |
||||
- [3af1aaa](https://github.com/grpc/grpc/commit/3af1aaadabf49bc6274711a11f81627c0f351a9a) |
||||
Basic setup to build gRPC Python with Bazel |
||||
- [11f199e](https://github.com/grpc/grpc/commit/11f199e34dc416a2bd8b56391b242a867bedade4) |
||||
Workspace changes to build gRPC Python with Bazel |
||||
- [848fd9d](https://github.com/grpc/grpc/commit/848fd9d75f6df10f00e8328ff052c0237b3002ab) |
||||
Minimal Bazel BUILD files for grpcio Python |
||||
|
||||
### Other contibutions |
||||
|
||||
- [89ce16b](https://github.com/grpc/grpc/commit/89ce16b6daaad4caeb1c9ba670c6c4b62ea1a93c) |
||||
Update Dockerfiles for python artifacts to use latest git version |
||||
- [32f7c48](https://github.com/grpc/grpc/commit/32f7c48dad71cac7af652bf994ab1dde3ddb0607) |
||||
Revert removals from python artifact dockerfiles |
||||
- [712eb9f](https://github.com/grpc/grpc/commit/712eb9ff91cde66af94e8381ec01ad512ed6d03c) |
||||
Make logging after success in jobset more apparent |
||||
- [c6e4372](https://github.com/grpc/grpc/commit/c6e4372f8a93bb0eb996b5f202465785422290f2) |
||||
Create README for gRPC Python reflection package |
||||
- [2e113ca](https://github.com/grpc/grpc/commit/2e113ca6b2cc31aa8a9687d40ee1bd759381654f) |
||||
Update logging in Python to use module-level logger |
||||
|
||||
### Pending PRs |
||||
|
||||
- BUILD files for all tests in |
||||
[tests.json](https://github.com/ghostwriternr/grpc/blob/70c8a58b2918a5369905e5a203d7ce7897b6207e/src/python/grpcio_tests/tests/tests.json). |
||||
- BUILD files for gRPC testing, gRPC health checking, gRPC reflection. |
||||
- (Yet to complete) BUILD files for grpcio_tools. One test depends on this. |
||||
|
||||
## Known issues |
||||
|
||||
- [grpc/grpc #16336](https://github.com/grpc/grpc/issues/16336) RuntimeError |
||||
for `_reconnect_test` Python unit test with Bazel |
||||
- Some tests in Bazel pass despite throwing an exception. Example: |
||||
`testAbortedStreamStream` in |
||||
`src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py`. |
||||
- [#14557](https://github.com/grpc/grpc/pull/14557) introduced a minor bug |
||||
where the module level loggers don't initialize a default logging handler. |
||||
- Sanity test doesn't make sense in the context of Bazel, and thus fails. |
||||
- There are some issues with Python2 vs Python3. Specifically, |
||||
- On some machines, “cygrpc.so: undefined symbol: _Py_FalseStruct” error |
||||
shows up. This is because of incorrect Python version being used to build |
||||
Cython. |
||||
- Some external packages like enum34 throw errors when used with Python 3 and |
||||
some extra packages are currently installed as Python version in current |
||||
build scripts. For now, the extra packages are added to a |
||||
`requirements.bazel.txt` file in the repository root. |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue