Merge pull request #7416 from soltanmm/3D

Build Python distributions for Python 3
pull/7374/head
kpayson64 8 years ago committed by GitHub
commit 7c467547ba
  1. 21
      setup.py
  2. 102
      src/python/grpcio/_unixccompiler_patch.py
  3. 27
      tools/distrib/python/grpcio_tools/setup.py
  4. 94
      tools/distrib/python/make_grpcio_tools.py
  5. 58
      tools/run_tests/artifact_targets.py
  6. 12
      tools/run_tests/build_artifact_python.bat

@ -47,10 +47,10 @@ from setuptools.command import egg_info
egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in'
PY3 = sys.version_info.major == 3
PYTHON_STEM = './src/python/grpcio'
CORE_INCLUDE = ('./include', '.',)
BORINGSSL_INCLUDE = ('./third_party/boringssl/include',)
ZLIB_INCLUDE = ('./third_party/zlib',)
PYTHON_STEM = os.path.join('src', 'python', 'grpcio')
CORE_INCLUDE = ('include', '.',)
BORINGSSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),)
ZLIB_INCLUDE = (os.path.join('third_party', 'zlib'),)
# Ensure we're in the proper directory whether or not we're being used by pip.
os.chdir(os.path.dirname(os.path.abspath(__file__)))
@ -62,8 +62,8 @@ import commands
import grpc_core_dependencies
import grpc_version
# TODO(atash) make this conditional on being on a mingw32 build
_unixccompiler_patch.monkeypatch_unix_compiler()
if 'win32' in sys.platform:
_unixccompiler_patch.monkeypatch_unix_compiler()
LICENSE = '3-clause BSD'
@ -105,7 +105,9 @@ if not "win32" in sys.platform:
if "win32" in sys.platform:
EXTENSION_LIBRARIES += ('ws2_32',)
DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
DEFINE_MACROS = (
('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600),
('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
if "win32" in sys.platform:
DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),)
if '64bit' in platform.architecture()[0]:
@ -200,12 +202,13 @@ COMMAND_CLASS = {
}
# Ensure that package data is copied over before any commands have been run:
credentials_dir = os.path.join(PYTHON_STEM, 'grpc/_cython/_credentials')
credentials_dir = os.path.join(PYTHON_STEM, 'grpc', '_cython', '_credentials')
try:
os.mkdir(credentials_dir)
except OSError:
pass
shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem'))
shutil.copyfile(os.path.join('etc', 'roots.pem'),
os.path.join(credentials_dir, 'roots.pem'))
PACKAGE_DATA = {
# Binaries that may or may not be present in the final installation, but are

@ -34,88 +34,44 @@ from distutils import errors
from distutils import unixccompiler
import os
import os.path
import shlex
import shutil
import sys
import tempfile
def _unix_commandfile_spawn(self, command):
"""Wrapper around distutils.util.spawn that attempts to use command files.
def _unix_piecemeal_link(
self, target_desc, objects, output_filename, output_dir=None,
libraries=None, library_dirs=None, runtime_library_dirs=None,
export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None,
build_temp=None, target_lang=None):
"""`link` externalized method taken almost verbatim from UnixCCompiler.
Meant to replace the CCompiler method `spawn` on UnixCCompiler and its
derivatives (e.g. the MinGW32 compiler).
Modifies the link command for unix-like compilers by using a command file so
that long command line argument strings don't break the command shell's
ARG_MAX character limit.
Some commands like `gcc` (and friends like `clang`) support command files to
work around shell command length limits.
"""
objects, output_dir = self._fix_object_args(objects, output_dir)
libraries, library_dirs, runtime_library_dirs = self._fix_lib_args(
libraries, library_dirs, runtime_library_dirs)
# filter out standard library paths, which are not explicitely needed
# for linking
library_dirs = [dir for dir in library_dirs
if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')]
runtime_library_dirs = [dir for dir in runtime_library_dirs
if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')]
lib_opts = ccompiler.gen_lib_options(self, library_dirs, runtime_library_dirs,
libraries)
if (not (isinstance(output_dir, str) or isinstance(output_dir, bytes))
and output_dir is not None):
raise TypeError("'output_dir' must be a string or None")
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
ld_args = (objects + self.objects +
lib_opts + ['-o', output_filename])
if debug:
ld_args[:0] = ['-g']
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
if target_desc == ccompiler.CCompiler.EXECUTABLE:
linker = self.linker_exe[:]
else:
linker = self.linker_so[:]
if target_lang == "c++" and self.compiler_cxx:
# skip over environment variable settings if /usr/bin/env
# is used to set up the linker's environment.
# This is needed on OSX. Note: this assumes that the
# normal and C++ compiler have the same environment
# settings.
i = 0
if os.path.basename(linker[0]) == "env":
i = 1
while '=' in linker[i]:
i = i + 1
linker[i] = self.compiler_cxx[i]
if sys.platform == 'darwin':
import _osx_support
linker = _osx_support.compiler_fixup(linker, ld_args)
temporary_directory = tempfile.mkdtemp()
command_filename = os.path.abspath(
os.path.join(temporary_directory, 'command'))
with open(command_filename, 'w') as command_file:
escaped_ld_args = [arg.replace('\\', '\\\\') for arg in ld_args]
command_file.write(' '.join(escaped_ld_args))
self.spawn(linker + ['@{}'.format(command_filename)])
except errors.DistutilsExecError:
raise ccompiler.LinkError
# Sometimes distutils embeds the executables as full strings including some
# hard-coded flags rather than as lists.
command = list(shlex.split(command[0])) + list(command[1:])
command_base = os.path.basename(command[0].strip())
if command_base == 'ccache':
command_base = command[:2]
command_args = command[2:]
elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']:
command_base = command[:1]
command_args = command[1:]
else:
log.debug("skipping %s (up-to-date)", output_filename)
return ccompiler.CCompiler.spawn(self, command)
temporary_directory = tempfile.mkdtemp()
command_filename = os.path.abspath(os.path.join(temporary_directory, 'command'))
with open(command_filename, 'w') as command_file:
escaped_args = [arg.replace('\\', '\\\\') for arg in command_args]
command_file.write(' '.join(escaped_args))
modified_command = command_base + ['@{}'.format(command_filename)]
result = ccompiler.CCompiler.spawn(self, modified_command)
shutil.rmtree(temporary_directory)
return result
# TODO(atash) try replacing this monkeypatch of the compiler harness' link
# operation with a monkeypatch of the distutils `spawn` that applies
# command-argument-file hacks where it can. Might be cleaner.
def monkeypatch_unix_compiler():
"""Monkeypatching is dumb, but it's either that or we become maintainers of
something much, much bigger."""
unixccompiler.UnixCCompiler.link = _unix_piecemeal_link
unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn

@ -61,6 +61,13 @@ EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
'-lpthread'))
CC_FILES = [
os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
PROTO_FILES = [
os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES]
CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE)
PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
@ -82,8 +89,8 @@ def package_data():
proto_resources_path = os.path.join(tools_path,
GRPC_PYTHON_PROTO_RESOURCES_NAME)
proto_files = []
for proto_file in protoc_lib_deps.PROTO_FILES:
source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file)
for proto_file in PROTO_FILES:
source = os.path.join(PROTO_INCLUDE, proto_file)
target = os.path.join(proto_resources_path, proto_file)
relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file)
try:
@ -99,18 +106,20 @@ def package_data():
def protoc_ext_module():
plugin_sources = [
'grpc/tools/main.cc',
'grpc_root/src/compiler/python_generator.cc'] + [
os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
for cc_file in protoc_lib_deps.CC_FILES]
os.path.join('grpc', 'tools', 'main.cc'),
os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [
os.path.join(CC_INCLUDE, cc_file)
for cc_file in CC_FILES]
plugin_ext = extension.Extension(
name='grpc.tools._protoc_compiler',
sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources,
sources=(
[os.path.join('grpc', 'tools', '_protoc_compiler.pyx')] +
plugin_sources),
include_dirs=[
'.',
'grpc_root',
'grpc_root/include',
protoc_lib_deps.CC_INCLUDE,
os.path.join('grpc_root', 'include'),
CC_INCLUDE,
],
language='c++',
define_macros=list(DEFINE_MACROS),

@ -29,12 +29,18 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import errno
import filecmp
import glob
import os
import os.path
import shutil
import subprocess
import sys
import traceback
import uuid
DEPS_FILE_CONTENT="""
# Copyright 2016, Google Inc.
@ -124,20 +130,88 @@ def get_deps():
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
return deps_file_content
def long_path(path):
if os.name == 'nt':
return '\\\\?\\' + path
else:
return path
def atomic_file_copy(src, dst):
"""Based on the lock-free-whack-a-mole algorithm, depending on filesystem
renaming being atomic. Described at http://stackoverflow.com/a/28090883.
"""
try:
if filecmp.cmp(src, dst):
return
except:
pass
dst_dir = os.path.abspath(os.path.dirname(dst))
dst_base = os.path.basename(dst)
this_id = str(uuid.uuid4()).replace('.', '-')
temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
src = long_path(src)
dst = long_path(dst)
temporary_file = long_path(temporary_file)
mole_file = long_path(mole_file)
mole_pattern = long_path(mole_pattern)
shutil.copy2(src, temporary_file)
try:
os.rename(temporary_file, mole_file)
except:
print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
raise
for other_file in glob.glob(mole_pattern):
other_id = other_file.split('.')[-3]
if this_id == other_id:
pass
elif this_id < other_id:
try:
os.remove(other_file)
except:
pass
else:
try:
os.remove(mole_file)
except:
pass
this_id = other_id
mole_file = other_file
try:
if filecmp.cmp(src, dst):
try:
os.remove(mole_file)
except:
pass
return
except:
pass
try:
os.rename(mole_file, dst)
except:
pass
def main():
os.chdir(GRPC_ROOT)
for tree in [GRPC_PYTHON_PROTOBUF,
GRPC_PYTHON_PROTOC_PLUGINS,
GRPC_PYTHON_INCLUDE]:
try:
shutil.rmtree(tree)
except Exception as _:
pass
shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF)
shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
for source, target in [
(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF),
(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS),
(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]:
for source_dir, _, files in os.walk(source):
target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source)))
try:
os.makedirs(target_dir)
except OSError as error:
if error.errno != errno.EEXIST:
raise
for relative_file in files:
source_file = os.path.abspath(os.path.join(source_dir, relative_file))
target_file = os.path.abspath(os.path.join(target_dir, relative_file))
atomic_file_copy(source_file, target_file)
try:
protoc_lib_deps_content = get_deps()

@ -30,11 +30,14 @@
"""Definition of targets to build artifacts."""
import os.path
import sys
import jobset
def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
flake_retries=0, timeout_retries=0):
flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args,
environ=docker_env,
shortname='build_artifact.%s' % (name),
timeout_seconds=30*60,
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries)
return jobspec
def create_jobspec(name, cmdline, environ=None, shell=False,
flake_retries=0, timeout_retries=0):
flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec."""
jobspec = jobset.JobSpec(
cmdline=cmdline,
environ=environ,
shortname='build_artifact.%s' % (name),
timeout_seconds=30*60,
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell)
@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = {
'x64': '-m64'
}
python_version_arch_map = {
'x86': 'Python27_32bits',
'x64': 'Python27'
python_windows_version_arch_map = {
('x86', '2.7'): 'Python27_32bits',
('x64', '2.7'): 'Python27',
('x86', '3.4'): 'Python34_32bits',
('x64', '3.4'): 'Python34',
}
class PythonArtifact:
"""Builds Python artifacts."""
def __init__(self, platform, arch, manylinux_build=None):
def __init__(self, platform, arch, python_version, manylinux_build=None):
if manylinux_build:
self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build)
self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
else:
self.name = 'python_%s_%s' % (platform, arch)
self.name = 'python%s_%s_%s' % (python_version, platform, arch)
self.platform = platform
self.arch = arch
self.labels = ['artifact', 'python', platform, arch]
self.python_version = python_version_arch_map[arch]
self.labels = ['artifact', 'python', python_version, platform, arch]
self.python_version = python_version
self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
self.manylinux_build = manylinux_build
def pre_build_jobspecs(self):
return []
return []
def build_jobspec(self):
environ = {}
@ -107,7 +113,6 @@ class PythonArtifact:
# special places...
environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
# Our docker image has all the prerequisites pip-installed already.
environ['SKIP_PIP_INSTALL'] = '1'
# Platform autodetection for the manylinux1 image breaks so we set the
# defines ourselves.
@ -117,16 +122,18 @@ class PythonArtifact:
return create_docker_jobspec(self.name,
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
'tools/run_tests/build_artifact_python.sh',
environ=environ)
environ=environ,
timeout_seconds=60*60)
elif self.platform == 'windows':
return create_jobspec(self.name,
['tools\\run_tests\\build_artifact_python.bat',
self.python_version,
self.python_windows_prefix,
'32' if self.arch == 'x86' else '64'
],
shell=True)
else:
environ['SKIP_PIP_INSTALL'] = 'TRUE'
environ['PYTHON'] = 'python{}'.format(self.python_version)
return create_jobspec(self.name,
['tools/run_tests/build_artifact_python.sh'],
environ=environ)
@ -323,13 +330,18 @@ def targets():
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
for platform in ('linux', 'macos', 'windows')
for arch in ('x86', 'x64')] +
[PythonArtifact('linux', 'x86', 'cp27-cp27m'),
PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
PythonArtifact('linux', 'x64', 'cp27-cp27m'),
PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
PythonArtifact('macos', 'x64'),
PythonArtifact('windows', 'x86'),
PythonArtifact('windows', 'x64'),
[PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
PythonArtifact('macos', 'x64', '2.7'),
PythonArtifact('windows', 'x86', '2.7'),
PythonArtifact('windows', 'x64', '2.7'),
PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
PythonArtifact('macos', 'x64', '3.4'),
PythonArtifact('windows', 'x86', '3.4'),
PythonArtifact('windows', 'x64', '3.4'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'),

@ -65,11 +65,19 @@ python tools\distrib\python\make_grpcio_tools.py
@rem Build gRPC Python extensions
python setup.py build_ext -c mingw32
python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
pushd tools\distrib\python\grpcio_tools
python setup.py build_ext -c mingw32
popd
@rem Build gRPC Python distributions
python setup.py bdist_wheel
python tools\distrib\python\grpcio_tools\setup.py bdist_wheel
pushd tools\distrib\python\grpcio_tools
python setup.py bdist_wheel
popd
mkdir artifacts
xcopy /Y /I /S dist\* artifacts\ || goto :error

Loading…
Cancel
Save