grpc/tools/distrib/python/make_grpcio_tools.py

#!/usr/bin/env python

# Copyright 2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

import errno
import filecmp
import glob
import os
import os.path
import shutil
import subprocess
import sys
import traceback
import uuid

DEPS_FILE_CONTENT="""
# Copyright 2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# AUTO-GENERATED BY make_grpcio_tools.py!
CC_FILES={cc_files}
PROTO_FILES={proto_files}

CC_INCLUDE={cc_include}
PROTO_INCLUDE={proto_include}
"""

# Bazel query result prefix for expected source files in protobuf.
PROTOBUF_CC_PREFIX = '//:src/'
PROTOBUF_PROTO_PREFIX = '//:src/'

GRPC_ROOT = os.path.abspath(
    os.path.join(os.path.dirname(os.path.abspath(__file__)),
                 '..', '..', '..'))

GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools')

GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = 'third_party/protobuf/src'
GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler')
GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
                                    'third_party/protobuf/src')
GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT,
                                          'grpc_root/src/compiler')
GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT,
                                           'protoc_lib_deps.py')

GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include')
GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')

BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'


def bazel_query(query):
  output = subprocess.check_output([BAZEL_DEPS, query])
  return output.splitlines()

def get_deps():
  """Write the result of the bazel query `query` against protobuf to
     `out_file`."""
  cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY)
  cc_files = [
      name[len(PROTOBUF_CC_PREFIX):] for name in cc_files_output
      if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)]
  proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY)
  proto_files = [
      name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
      if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
  deps_file_content = DEPS_FILE_CONTENT.format(
      cc_files=cc_files,
      proto_files=proto_files,
      cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
  return deps_file_content

def long_path(path):
  if os.name == 'nt':
    return '\\\\?\\' + path
  else:
    return path

def atomic_file_copy(src, dst):
  """Based on the lock-free-whack-a-mole algorithm, depending on filesystem
     renaming being atomic. Described at http://stackoverflow.com/a/28090883.
  """
  try:
    if filecmp.cmp(src, dst):
      return
  except:
    pass
  dst_dir = os.path.abspath(os.path.dirname(dst))
  dst_base = os.path.basename(dst)
  this_id = str(uuid.uuid4()).replace('.', '-')
  temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
  mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
  mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
  src = long_path(src)
  dst = long_path(dst)
  temporary_file = long_path(temporary_file)
  mole_file = long_path(mole_file)
  mole_pattern = long_path(mole_pattern)
  shutil.copy2(src, temporary_file)
  try:
    os.rename(temporary_file, mole_file)
  except:
    print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
    print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
    raise
  for other_file in glob.glob(mole_pattern):
    other_id = other_file.split('.')[-3]
    if this_id == other_id:
      pass
    elif this_id < other_id:
      try:
        os.remove(other_file)
      except:
        pass
    else:
      try:
        os.remove(mole_file)
      except:
        pass
      this_id = other_id
      mole_file = other_file
  try:
    if filecmp.cmp(src, dst):
      try:
        os.remove(mole_file)
      except:
        pass
      return
  except:
    pass
  try:
    os.rename(mole_file, dst)
  except:
    pass


def main():
  os.chdir(GRPC_ROOT)

  for source, target in [
      (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF),
      (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS),
      (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]:
    for source_dir, _, files in os.walk(source):
      target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source)))
      try:
        os.makedirs(target_dir)
      except OSError as error:
        if error.errno != errno.EEXIST:
          raise
      for relative_file in files:
        source_file = os.path.abspath(os.path.join(source_dir, relative_file))
        target_file = os.path.abspath(os.path.join(target_dir, relative_file))
        atomic_file_copy(source_file, target_file)

  try:
    protoc_lib_deps_content = get_deps()
  except Exception as error:
    # We allow this script to succeed even if we couldn't get the dependencies,
    # as then we can assume that even without a successful bazel run the
    # dependencies currently in source control are 'good enough'.
    sys.stderr.write("Got non-fatal error:\n")
    traceback.print_exc(file=sys.stderr)
    return
  # If we successfully got the dependencies, truncate and rewrite the deps file.
  with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file:
    deps_file.write(protoc_lib_deps_content)

if __name__ == '__main__':
  main()