mirror of https://github.com/grpc/grpc.git
This commit resolves #18331. This commit resolves #18256. This commit resolves... another TODO that apparently didn't have an associated github issue. We swap out pubref's implementation of py_proto_library with our own, which more closely mirrors the interface of the internal py_proto_library, taking the descriptor file output of a proto_library rule as input. One minor change in behavior was introduced for simplicity. When a py_proto_library depends on a proto_library with a source proto file in a subdirectory of the bazel package, the import module of the resultant python library will reflect the package, *not* the full directory of the proto file, including both the bazel package and the subdirectories, as pubref did previously. This behavior also more closely mirrors google internal behavior. This commit also introduces a slightly more stringent bazel format script. Buildifier on its own will not take care of long lines, but by running yapf first, we end up with a more legible file. At the moment, there is no sanity check associated with this formatter.pull/18872/head
Richard Belleville
6 years ago
parent
10e39e316c
commit
05f37c8143
23 changed files with 545 additions and 198 deletions
@ -1,16 +1,8 @@ |
||||
load("//third_party/py:python_configure.bzl", "python_configure") |
||||
load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") |
||||
load("@grpc_python_dependencies//:requirements.bzl", "pip_install") |
||||
load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories") |
||||
|
||||
def grpc_python_deps(): |
||||
# TODO(https://github.com/grpc/grpc/issues/18256): Remove conditional. |
||||
if hasattr(native, "http_archive"): |
||||
python_configure(name = "local_config_python") |
||||
pip_repositories() |
||||
pip_install() |
||||
py_proto_repositories() |
||||
else: |
||||
print("Building Python gRPC with bazel 23.0+ is disabled pending " + |
||||
"resolution of https://github.com/grpc/grpc/issues/18256.") |
||||
|
||||
python_configure(name = "local_config_python") |
||||
pip_repositories() |
||||
pip_install() |
||||
|
||||
@ -0,0 +1,84 @@ |
||||
"""Utility functions for generating protobuf code.""" |
||||
|
||||
_PROTO_EXTENSION = ".proto" |
||||
|
||||
def get_proto_root(workspace_root): |
||||
"""Gets the root protobuf directory. |
||||
|
||||
Args: |
||||
workspace_root: context.label.workspace_root |
||||
|
||||
Returns: |
||||
The directory relative to which generated include paths should be. |
||||
""" |
||||
if workspace_root: |
||||
return "/{}".format(workspace_root) |
||||
else: |
||||
return "" |
||||
|
||||
def _strip_proto_extension(proto_filename): |
||||
if not proto_filename.endswith(_PROTO_EXTENSION): |
||||
fail('"{}" does not end with "{}"'.format( |
||||
proto_filename, |
||||
_PROTO_EXTENSION, |
||||
)) |
||||
return proto_filename[:-len(_PROTO_EXTENSION)] |
||||
|
||||
def proto_path_to_generated_filename(proto_path, fmt_str): |
||||
"""Calculates the name of a generated file for a protobuf path. |
||||
|
||||
For example, "examples/protos/helloworld.proto" might map to |
||||
"helloworld.pb.h". |
||||
|
||||
Args: |
||||
proto_path: The path to the .proto file. |
||||
fmt_str: A format string used to calculate the generated filename. For |
||||
example, "{}.pb.h" might be used to calculate a C++ header filename. |
||||
|
||||
Returns: |
||||
The generated filename. |
||||
""" |
||||
return fmt_str.format(_strip_proto_extension(proto_path)) |
||||
|
||||
def _get_include_directory(include): |
||||
directory = include.path |
||||
if directory.startswith("external"): |
||||
external_separator = directory.find("/") |
||||
repository_separator = directory.find("/", external_separator + 1) |
||||
return directory[:repository_separator] |
||||
else: |
||||
return "." |
||||
|
||||
def get_include_protoc_args(includes): |
||||
"""Returns protoc args that imports protos relative to their import root. |
||||
|
||||
Args: |
||||
includes: A list of included proto files. |
||||
|
||||
Returns: |
||||
A list of arguments to be passed to protoc. For example, ["--proto_path=."]. |
||||
""" |
||||
return [ |
||||
"--proto_path={}".format(_get_include_directory(include)) |
||||
for include in includes |
||||
] |
||||
|
||||
def get_plugin_args(plugin, flags, dir_out, generate_mocks): |
||||
"""Returns arguments configuring protoc to use a plugin for a language. |
||||
|
||||
Args: |
||||
plugin: An executable file to run as the protoc plugin. |
||||
flags: The plugin flags to be passed to protoc. |
||||
dir_out: The output directory for the plugin. |
||||
generate_mocks: A bool indicating whether to generate mocks. |
||||
|
||||
Returns: |
||||
A list of protoc arguments configuring the plugin. |
||||
""" |
||||
augmented_flags = list(flags) |
||||
if generate_mocks: |
||||
augmented_flags.append("generate_mock_code=true") |
||||
return [ |
||||
"--plugin=protoc-gen-PLUGIN=" + plugin.path, |
||||
"--PLUGIN_out=" + ",".join(augmented_flags) + ":" + dir_out, |
||||
] |
||||
@ -0,0 +1,203 @@ |
||||
"""Generates and compiles Python gRPC stubs from proto_library rules.""" |
||||
|
||||
load("@grpc_python_dependencies//:requirements.bzl", "requirement") |
||||
load( |
||||
"//bazel:protobuf.bzl", |
||||
"get_include_protoc_args", |
||||
"get_plugin_args", |
||||
"get_proto_root", |
||||
"proto_path_to_generated_filename", |
||||
) |
||||
|
||||
_GENERATED_PROTO_FORMAT = "{}_pb2.py" |
||||
_GENERATED_GRPC_PROTO_FORMAT = "{}_pb2_grpc.py" |
||||
|
||||
def _get_staged_proto_file(context, source_file): |
||||
if source_file.dirname == context.label.package: |
||||
return source_file |
||||
else: |
||||
copied_proto = context.actions.declare_file(source_file.basename) |
||||
context.actions.run_shell( |
||||
inputs = [source_file], |
||||
outputs = [copied_proto], |
||||
command = "cp {} {}".format(source_file.path, copied_proto.path), |
||||
mnemonic = "CopySourceProto", |
||||
) |
||||
return copied_proto |
||||
|
||||
def _generate_py_impl(context): |
||||
protos = [] |
||||
for src in context.attr.deps: |
||||
for file in src.proto.direct_sources: |
||||
protos.append(_get_staged_proto_file(context, file)) |
||||
includes = [ |
||||
file |
||||
for src in context.attr.deps |
||||
for file in src.proto.transitive_imports |
||||
] |
||||
proto_root = get_proto_root(context.label.workspace_root) |
||||
format_str = (_GENERATED_GRPC_PROTO_FORMAT if context.executable.plugin else _GENERATED_PROTO_FORMAT) |
||||
out_files = [ |
||||
context.actions.declare_file( |
||||
proto_path_to_generated_filename( |
||||
proto.basename, |
||||
format_str, |
||||
), |
||||
) |
||||
for proto in protos |
||||
] |
||||
|
||||
arguments = [] |
||||
tools = [context.executable._protoc] |
||||
if context.executable.plugin: |
||||
arguments += get_plugin_args( |
||||
context.executable.plugin, |
||||
context.attr.flags, |
||||
context.genfiles_dir.path, |
||||
False, |
||||
) |
||||
tools += [context.executable.plugin] |
||||
else: |
||||
arguments += [ |
||||
"--python_out={}:{}".format( |
||||
",".join(context.attr.flags), |
||||
context.genfiles_dir.path, |
||||
), |
||||
] |
||||
|
||||
arguments += get_include_protoc_args(includes) |
||||
arguments += [ |
||||
"--proto_path={}".format(context.genfiles_dir.path) |
||||
for proto in protos |
||||
] |
||||
for proto in protos: |
||||
massaged_path = proto.path |
||||
if massaged_path.startswith(context.genfiles_dir.path): |
||||
massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] |
||||
arguments.append(massaged_path) |
||||
|
||||
well_known_proto_files = [] |
||||
if context.attr.well_known_protos: |
||||
well_known_proto_directory = context.attr.well_known_protos.files.to_list( |
||||
)[0].dirname |
||||
|
||||
arguments += ["-I{}".format(well_known_proto_directory + "/../..")] |
||||
well_known_proto_files = context.attr.well_known_protos.files.to_list() |
||||
|
||||
context.actions.run( |
||||
inputs = protos + includes + well_known_proto_files, |
||||
tools = tools, |
||||
outputs = out_files, |
||||
executable = context.executable._protoc, |
||||
arguments = arguments, |
||||
mnemonic = "ProtocInvocation", |
||||
) |
||||
return struct(files = depset(out_files)) |
||||
|
||||
__generate_py = rule( |
||||
attrs = { |
||||
"deps": attr.label_list( |
||||
mandatory = True, |
||||
allow_empty = False, |
||||
providers = ["proto"], |
||||
), |
||||
"plugin": attr.label( |
||||
executable = True, |
||||
providers = ["files_to_run"], |
||||
cfg = "host", |
||||
), |
||||
"flags": attr.string_list( |
||||
mandatory = False, |
||||
allow_empty = True, |
||||
), |
||||
"well_known_protos": attr.label(mandatory = False), |
||||
"_protoc": attr.label( |
||||
default = Label("//external:protocol_compiler"), |
||||
executable = True, |
||||
cfg = "host", |
||||
), |
||||
}, |
||||
output_to_genfiles = True, |
||||
implementation = _generate_py_impl, |
||||
) |
||||
|
||||
def _generate_py(well_known_protos, **kwargs): |
||||
if well_known_protos: |
||||
__generate_py( |
||||
well_known_protos = "@com_google_protobuf//:well_known_protos", |
||||
**kwargs |
||||
) |
||||
else: |
||||
__generate_py(**kwargs) |
||||
|
||||
_WELL_KNOWN_PROTO_LIBS = [ |
||||
"@com_google_protobuf//:any_proto", |
||||
"@com_google_protobuf//:api_proto", |
||||
"@com_google_protobuf//:compiler_plugin_proto", |
||||
"@com_google_protobuf//:descriptor_proto", |
||||
"@com_google_protobuf//:duration_proto", |
||||
"@com_google_protobuf//:empty_proto", |
||||
"@com_google_protobuf//:field_mask_proto", |
||||
"@com_google_protobuf//:source_context_proto", |
||||
"@com_google_protobuf//:struct_proto", |
||||
"@com_google_protobuf//:timestamp_proto", |
||||
"@com_google_protobuf//:type_proto", |
||||
"@com_google_protobuf//:wrappers_proto", |
||||
] |
||||
|
||||
def py_proto_library( |
||||
name, |
||||
deps, |
||||
well_known_protos = True, |
||||
proto_only = False, |
||||
**kwargs): |
||||
"""Generate python code for a protobuf. |
||||
|
||||
Args: |
||||
name: The name of the target. |
||||
deps: A list of dependencies. Must contain a single element. |
||||
well_known_protos: A bool indicating whether or not to include well-known |
||||
protos. |
||||
proto_only: A bool indicating whether to generate vanilla protobuf code |
||||
or to also generate gRPC code. |
||||
""" |
||||
if len(deps) > 1: |
||||
fail("The supported length of 'deps' is 1.") |
||||
|
||||
codegen_target = "_{}_codegen".format(name) |
||||
codegen_grpc_target = "_{}_grpc_codegen".format(name) |
||||
|
||||
well_known_proto_rules = _WELL_KNOWN_PROTO_LIBS if well_known_protos else [] |
||||
|
||||
_generate_py( |
||||
name = codegen_target, |
||||
deps = deps, |
||||
well_known_protos = well_known_protos, |
||||
**kwargs |
||||
) |
||||
|
||||
if not proto_only: |
||||
_generate_py( |
||||
name = codegen_grpc_target, |
||||
deps = deps, |
||||
plugin = "//:grpc_python_plugin", |
||||
well_known_protos = well_known_protos, |
||||
**kwargs |
||||
) |
||||
|
||||
native.py_library( |
||||
name = name, |
||||
srcs = [ |
||||
":{}".format(codegen_grpc_target), |
||||
":{}".format(codegen_target), |
||||
], |
||||
deps = [requirement("protobuf")], |
||||
**kwargs |
||||
) |
||||
else: |
||||
native.py_library( |
||||
name = name, |
||||
srcs = [":{}".format(codegen_target), ":{}".format(codegen_target)], |
||||
deps = [requirement("protobuf")], |
||||
**kwargs |
||||
) |
||||
@ -1,30 +1,32 @@ |
||||
load("@grpc_python_dependencies//:requirements.bzl", "requirement") |
||||
load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") |
||||
|
||||
package(default_visibility = ["//visibility:public"]) |
||||
load("//bazel:python_rules.bzl", "py_proto_library") |
||||
|
||||
proto_library( |
||||
name = "empty2_proto_descriptor", |
||||
srcs = ["empty2.proto"], |
||||
) |
||||
|
||||
py_proto_library( |
||||
name = "empty2_proto", |
||||
protos = [ |
||||
"empty2.proto", |
||||
], |
||||
with_grpc = True, |
||||
deps = [ |
||||
requirement('protobuf'), |
||||
":empty2_proto_descriptor", |
||||
], |
||||
) |
||||
|
||||
proto_library( |
||||
name = "empty2_extensions_proto_descriptor", |
||||
srcs = ["empty2_extensions.proto"], |
||||
deps = [ |
||||
":empty2_proto_descriptor", |
||||
] |
||||
) |
||||
|
||||
py_proto_library( |
||||
name = "empty2_extensions_proto", |
||||
protos = [ |
||||
"empty2_extensions.proto", |
||||
], |
||||
proto_deps = [ |
||||
":empty2_proto", |
||||
], |
||||
with_grpc = True, |
||||
deps = [ |
||||
requirement('protobuf'), |
||||
":empty2_extensions_proto_descriptor", |
||||
], |
||||
) |
||||
|
||||
|
||||
@ -0,0 +1,4 @@ |
||||
[style] |
||||
based_on_style = google |
||||
allow_split_before_dict_value = False |
||||
spaces_around_default_or_named_assign = True |
||||
@ -0,0 +1,39 @@ |
||||
#!/bin/bash |
||||
# Copyright 2019 The gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set=-ex |
||||
|
||||
VIRTUAL_ENV=bazel_format_virtual_environment |
||||
|
||||
CONFIG_PATH="$(dirname ${0})/bazel_style.cfg" |
||||
|
||||
python -m virtualenv ${VIRTUAL_ENV} |
||||
PYTHON=${VIRTUAL_ENV}/bin/python |
||||
"$PYTHON" -m pip install --upgrade pip==10.0.1 |
||||
"$PYTHON" -m pip install --upgrade futures |
||||
"$PYTHON" -m pip install yapf==0.20.0 |
||||
|
||||
pushd "$(dirname "${0}")/../.." |
||||
FILES=$(find . -path ./third_party -prune -o -name '*.bzl' -print) |
||||
echo "${FILES}" | xargs "$PYTHON" -m yapf -i --style="${CONFIG_PATH}" |
||||
|
||||
if ! which buildifier &>/dev/null; then |
||||
echo 'buildifer must be installed.' >/dev/stderr |
||||
exit 1 |
||||
fi |
||||
|
||||
echo "${FILES}" | xargs buildifier --type=bzl |
||||
|
||||
popd |
||||
Loading…
Reference in new issue