grpc/tools/distrib/python/make_grpcio_tools.py

#!/usr/bin/env python3

# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import errno
import filecmp
import glob
import os
import os.path
import pprint
import shutil
import subprocess
import sys
import traceback
import uuid

# the template for the content of protoc_lib_deps.py
DEPS_FILE_CONTENT = """
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# AUTO-GENERATED BY make_grpcio_tools.py!
CC_FILES={cc_files}

PROTO_FILES={proto_files}

CC_INCLUDES={cc_includes}
PROTO_INCLUDE={proto_include}

{commit_hash_expr}
"""

# expose commit hash suffix and prefix for check_grpcio_tools.py
COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="'
COMMIT_HASH_SUFFIX = '"'

EXTERNAL_LINKS = [
    ('@com_google_absl//', 'third_party/abseil-cpp/'),
    ('@com_google_protobuf//', 'third_party/protobuf/'),
    ('@utf8_range//:', 'third_party/utf8_range/'),
]

PROTOBUF_PROTO_PREFIX = '@com_google_protobuf//src/'

# will be added to include path when building grpcio_tools
CC_INCLUDES = [
    os.path.join('third_party', 'abseil-cpp'),
    os.path.join('third_party', 'protobuf', 'src'),
    os.path.join('third_party', 'utf8_range'),
]

# include path for .proto files
PROTO_INCLUDE = os.path.join('third_party', 'protobuf', 'src')

# the target directory is relative to the grpcio_tools package root.
GRPCIO_TOOLS_ROOT_PREFIX = 'tools/distrib/python/grpcio_tools/'

# Pairs of (source, target) directories to copy
# from the grpc repo root to the grpcio_tools build root.
COPY_FILES_SOURCE_TARGET_PAIRS = [
    ('include', 'grpc_root/include'),
    ('src/compiler', 'grpc_root/src/compiler'),
    ('third_party/abseil-cpp/absl', 'third_party/abseil-cpp/absl'),
    ('third_party/protobuf/src', 'third_party/protobuf/src'),
    ('third_party/utf8_range', 'third_party/utf8_range')
]

# grpc repo root
GRPC_ROOT = os.path.abspath(
    os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..'))

# the directory under which to probe for the current protobuf commit SHA
GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party',
                                            'protobuf')

# the file to generate
GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib',
                                           'python', 'grpcio_tools',
                                           'protoc_lib_deps.py')

# the script to run for getting dependencies
BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python',
                          'bazel_deps.sh')

# the bazel target to scrape to get list of sources for the build
BAZEL_DEPS_PROTOC_LIB_QUERY = '@com_google_protobuf//:protoc_lib'

BAZEL_DEPS_COMMON_PROTOS_QUERIES = [
    '@com_google_protobuf//:well_known_type_protos',
    # has both plugin.proto and descriptor.proto
    '@com_google_protobuf//:compiler_plugin_proto',
]


def protobuf_submodule_commit_hash():
    """Gets the commit hash for the HEAD of the protobuf submodule currently
     checked out."""
    cwd = os.getcwd()
    os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT)
    output = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
    os.chdir(cwd)
    return output.decode("ascii").splitlines()[0].strip()


def _bazel_query(query):
    """Runs 'bazel query' to collect source file info."""
    print('Running "bazel query %s"' % query)
    output = subprocess.check_output([BAZEL_DEPS, query])
    return output.decode("ascii").splitlines()


def _pretty_print_list(items):
    """Pretty print python list"""
    formatted = pprint.pformat(items, indent=4)
    # add newline after opening bracket (and fix indent of the next line)
    if formatted.startswith('['):
        formatted = formatted[0] + '\n ' + formatted[1:]
    # add newline before closing bracket
    if formatted.endswith(']'):
        formatted = formatted[:-1] + '\n' + formatted[-1]
    return formatted


def _bazel_name_to_file_path(name):
    """Transform bazel reference to source file name."""
    for link in EXTERNAL_LINKS:
        if name.startswith(link[0]):
            filepath = link[1] + name[len(link[0]):].replace(':', '/')

            # For some reason, the WKT sources (such as wrappers.pb.cc)
            # end up being reported by bazel as having an extra 'wkt/google/protobuf'
            # in path. Removing it makes the compilation pass.
            # TODO(jtattermusch) Get dir of this hack.
            return filepath.replace('wkt/google/protobuf/', '')
    return None


def _generate_deps_file_content():
    """Returns the data structure with dependencies of protoc as python code."""
    cc_files_output = _bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY)

    # Collect .cc files (that will be later included in the native extension build)
    cc_files = []
    for name in cc_files_output:
        if name.endswith('.cc'):
            filepath = _bazel_name_to_file_path(name)
            if filepath:
                cc_files.append(filepath)

    # Collect list of .proto files that will be bundled in the grpcio_tools package.
    raw_proto_files = []
    for target in BAZEL_DEPS_COMMON_PROTOS_QUERIES:
        raw_proto_files += _bazel_query(target)
    proto_files = [
        name[len(PROTOBUF_PROTO_PREFIX):].replace(':', '/')
        for name in raw_proto_files
        if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)
    ]

    commit_hash = protobuf_submodule_commit_hash()
    commit_hash_expr = COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX

    deps_file_content = DEPS_FILE_CONTENT.format(
        cc_files=_pretty_print_list(sorted(cc_files)),
        proto_files=_pretty_print_list(sorted(set(proto_files))),
        cc_includes=_pretty_print_list(CC_INCLUDES),
        proto_include=repr(PROTO_INCLUDE),
        commit_hash_expr=commit_hash_expr)
    return deps_file_content


def _copy_source_tree(source, target):
    """Copies source directory to a given target directory."""
    print('Copying contents of %s to %s' % (source, target))
    # TODO(jtattermusch): It is unclear why this legacy code needs to copy
    # the source directory to the target via the following boilerplate.
    # Should this code be simplified?
    for source_dir, _, files in os.walk(source):
        target_dir = os.path.abspath(
            os.path.join(target, os.path.relpath(source_dir, source)))
        try:
            os.makedirs(target_dir)
        except OSError as error:
            if error.errno != errno.EEXIST:
                raise
        for relative_file in files:
            source_file = os.path.abspath(
                os.path.join(source_dir, relative_file))
            target_file = os.path.abspath(
                os.path.join(target_dir, relative_file))
            shutil.copyfile(source_file, target_file)


def main():
    os.chdir(GRPC_ROOT)

    # Step 1:
    # In order to be able to build the grpcio_tools package, we need the source code for the codegen plugins
    # and its dependencies to be available under the build root of the grpcio_tools package.
    # So we simply copy all the necessary files where the build will expect them to be.
    for source, target in COPY_FILES_SOURCE_TARGET_PAIRS:
        # convert the slashes in the relative path to platform-specific path dividers.
        # All paths are relative to GRPC_ROOT
        source_abs = os.path.join(GRPC_ROOT, os.path.join(*source.split('/')))
        # for targets, add grpcio_tools root prefix
        target = GRPCIO_TOOLS_ROOT_PREFIX + target
        target_abs = os.path.join(GRPC_ROOT, os.path.join(*target.split('/')))

        _copy_source_tree(source_abs, target_abs)
    print(
        'The necessary source files were copied under the grpcio_tools package root.'
    )
    print()

    # Step 2:
    # Extract build metadata from bazel build (by running "bazel query")
    # and populate the protoc_lib_deps.py file with python-readable data structure
    # that will be used by grpcio_tools's setup.py (so it knows how to configure
    # the native build for the codegen plugin)
    try:
        print('Invoking "bazel query" to gather the protobuf dependencies.')
        protoc_lib_deps_content = _generate_deps_file_content()
    except Exception as error:
        # We allow this script to succeed even if we couldn't get the dependencies,
        # as then we can assume that even without a successful bazel run the
        # dependencies currently in source control are 'good enough'.
        sys.stderr.write("Got non-fatal error:\n")
        traceback.print_exc(file=sys.stderr)
        return
    # If we successfully got the dependencies, truncate and rewrite the deps file.
    with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file:
        deps_file.write(protoc_lib_deps_content)
    print('File "%s" updated.' % GRPC_PYTHON_PROTOC_LIB_DEPS)
    print('Done.')


if __name__ == '__main__':
    main()