Merge pull request #7314 An error occurred from soltanmm/spinning-flags

Set more build-flags in `setup.py`s

setup.py

@@ -84,9 +84,40 @@ ENABLE_CYTHON_TRACING = os.environ.get(
 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
 # We use these environment variables to thus get around that without locking
 # ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', ''))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', ''))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+  EXTRA_ENV_COMPILE_ARGS = '-fno-wrapv'
+  if 'win32' in sys.platform:
+    # We use define flags here and don't directly add to DEFINE_MACROS below to
+    # ensure that the expert user/builder has a way of turning it off (via the
+    # envvars) without adding yet more GRPC-specific envvars.
+    # See https://sourceforge.net/p/mingw-w64/bugs/363/
+    if '32' in platform.architecture()[0]:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+    else:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+  elif "linux" in sys.platform or "darwin" in sys.platform:
+    EXTRA_ENV_COMPILE_ARGS += ' -fvisibility=hidden'
+if EXTRA_ENV_LINK_ARGS is None:
+  EXTRA_ENV_LINK_ARGS = '-lpthread'
+  if 'win32' in sys.platform:
+    # TODO(atash) check if this is actually safe to just import and call on
+    # non-Windows (to avoid breaking import style)
+    from distutils.cygwinccompiler import get_msvcr
+    msvcr = get_msvcr()[0]
+    # TODO(atash) sift through the GCC specs to see if libstdc++ can have any
+    # influence on the linkage outcome on MinGW for non-C++ programs.
+    EXTRA_ENV_LINK_ARGS += (
+        ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+        '-static'.format(msvcr=msvcr))
+  elif "linux" in sys.platform:
+    EXTRA_ENV_LINK_ARGS += ' -Wl,-wrap,memcpy'
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
 
 CYTHON_EXTENSION_PACKAGE_NAMES = ()
 
@@ -118,13 +149,8 @@ if "win32" in sys.platform:
 
 LDFLAGS = tuple(EXTRA_LINK_ARGS)
 CFLAGS = tuple(EXTRA_COMPILE_ARGS)
-if "linux" in sys.platform:
-  LDFLAGS += ('-Wl,-wrap,memcpy',)
 if "linux" in sys.platform or "darwin" in sys.platform:
-  CFLAGS += ('-fvisibility=hidden',)
-
   pymodinit_type = 'PyObject*' if PY3 else 'void'
-
   pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type)
   DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
 
@@ -139,8 +165,13 @@ if 'darwin' in sys.platform and PY3:
     os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
 
 
-def cython_extensions(module_names, extra_sources, include_dirs,
-                      libraries, define_macros, build_with_cython=False):
+def cython_extensions():
+  module_names = list(CYTHON_EXTENSION_MODULE_NAMES)
+  extra_sources = list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES)
+  include_dirs = list(EXTENSION_INCLUDE_DIRECTORIES)
+  libraries = list(EXTENSION_LIBRARIES)
+  define_macros = list(DEFINE_MACROS)
+  build_with_cython = bool(BUILD_WITH_CYTHON)
   # Set compiler directives linetrace argument only if we care about tracing;
   # this is due to Cython having different behavior between linetrace being
   # False and linetrace being unset. See issue #5689.
@@ -181,11 +212,7 @@ def cython_extensions(module_names, extra_sources, include_dirs,
   else:
     return extensions
 
-CYTHON_EXTENSION_MODULES = cython_extensions(
-    list(CYTHON_EXTENSION_MODULE_NAMES),
-    list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES),
-    list(EXTENSION_INCLUDE_DIRECTORIES), list(EXTENSION_LIBRARIES),
-    list(DEFINE_MACROS), bool(BUILD_WITH_CYTHON))
+CYTHON_EXTENSION_MODULES = cython_extensions()
 
 PACKAGE_DIRECTORIES = {
     '': PYTHON_STEM,

tools/distrib/python/grpcio_tools/setup.py

@@ -51,15 +51,43 @@ import grpc_version
 
 PY3 = sys.version_info.major == 3
 
+# Environment variable to determine whether or not the Cython extension should
+# *use* Cython or use the generated C files. Note that this requires the C files
+# to have been generated by building first *with* Cython support.
+BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False)
+
 # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
 # We use these environment variables to thus get around that without locking
 # ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
-                                                '-fno-wrapv -frtti -std=c++11'))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
-                                             '-lpthread'))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+  EXTRA_ENV_COMPILE_ARGS = '-fno-wrapv -frtti -std=c++11'
+  if 'win32' in sys.platform:
+    # We use define flags here and don't directly add to DEFINE_MACROS below to
+    # ensure that the expert user/builder has a way of turning it off (via the
+    # envvars) without adding yet more GRPC-specific envvars.
+    # See https://sourceforge.net/p/mingw-w64/bugs/363/
+    if '32' in platform.architecture()[0]:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+    else:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+if EXTRA_ENV_LINK_ARGS is None:
+  EXTRA_ENV_LINK_ARGS = '-lpthread'
+  if 'win32' in sys.platform:
+    # TODO(atash) check if this is actually safe to just import and call on
+    # non-Windows (to avoid breaking import style)
+    from distutils.cygwinccompiler import get_msvcr
+    msvcr = get_msvcr()[0]
+    EXTRA_ENV_LINK_ARGS += (
+        ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+        '-static'.format(msvcr=msvcr))
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
 
 GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
 GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
@@ -97,15 +125,19 @@ def package_data():
     proto_files.append(relative_target)
   return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files}
 
-def protoc_ext_module():
-  plugin_sources = [
+def extension_modules():
+  if BUILD_WITH_CYTHON:
+    plugin_sources = ['grpc/tools/_protoc_compiler.pyx']
+  else:
+    plugin_sources = ['grpc/tools/_protoc_compiler.cpp']
+  plugin_sources += [
       'grpc/tools/main.cc',
       'grpc_root/src/compiler/python_generator.cc'] + [
       os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
       for cc_file in protoc_lib_deps.CC_FILES]
   plugin_ext = extension.Extension(
       name='grpc.tools._protoc_compiler',
-      sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources,
+      sources=plugin_sources,
       include_dirs=[
           '.',
           'grpc_root',
@@ -117,19 +149,18 @@ def protoc_ext_module():
       extra_compile_args=list(EXTRA_COMPILE_ARGS),
       extra_link_args=list(EXTRA_LINK_ARGS),
   )
-  return plugin_ext
-
-def maybe_cythonize(exts):
-  from Cython import Build
-  return Build.cythonize(exts)
+  extensions = [plugin_ext]
+  if BUILD_WITH_CYTHON:
+    from Cython import Build
+    return Build.cythonize(extensions)
+  else:
+    return extensions
 
 setuptools.setup(
   name='grpcio_tools',
   version=grpc_version.VERSION,
   license='3-clause BSD',
-  ext_modules=maybe_cythonize([
-      protoc_ext_module(),
-  ]),
+  ext_modules=extension_modules(),
   packages=setuptools.find_packages('.'),
   namespace_packages=['grpc'],
   install_requires=[

tools/run_tests/build_artifact_python.bat

@@ -34,29 +34,6 @@ pip install --upgrade six
 pip install --upgrade setuptools
 pip install -rrequirements.txt
 
-@rem Because this is windows and *everything seems to hate Windows* we have to
-@rem set all of these flags ourselves because Python won't help us (see the
-@rem setup.py of the grpcio_tools project).
-set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11
-
-@rem See https://sourceforge.net/p/mingw-w64/bugs/363/
-if %2 == 32 (
-  set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s
-) else (
-  set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime64 -D_timeb=__timeb64
-)
-
-@rem Further confusing things, MSYS2's mingw64 tries to dynamically link
-@rem libgcc, libstdc++, and winpthreads. We have to override this or our
-@rem extensions end up linking to MSYS2 DLLs, which the normal Python on
-@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default
-@rem behavior of linking msvcrt.dll as the C runtime library, which we need to
-@rem override so that Python's distutils doesn't link us against multiple C
-@rem runtimes.
-python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt
-set /p PYTHON_MSVCR=<temp.txt
-set GRPC_PYTHON_LDFLAGS=-static-libgcc -static-libstdc++ -mcrtdll=%PYTHON_MSVCR% -static -lpthread
-
 set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
 

tools/run_tests/build_python.sh

@@ -127,19 +127,6 @@ if [ $(is_linux) ]; then
     fi
   fi
 fi
-# TODO(atash) consider conceptualizing MinGW as a first-class platform and move
-# these flags into our `setup.py`s
-if [ $(is_mingw) ]; then
-  # We're on MinGW, and our CFLAGS and LDFLAGS will be eaten by the void. Use
-  # our work-around environment variables instead.
-  PYTHON_MSVCR=`$PYTHON -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])"`
-  export GRPC_PYTHON_LDFLAGS="-static-libgcc -static-libstdc++ -mcrtdll=$PYTHON_MSVCR -static -lpthread"
-  # See https://sourceforge.net/p/mingw-w64/bugs/363/
-  export GRPC_PYTHON_CFLAGS="-D_ftime=_ftime64 -D_timeb=__timeb64"
-  # TODO(atash) set these flags for only grpcio-tools (they don't do any harm to
-  # grpcio, but they result in noisy warnings).
-  export GRPC_PYTHON_CFLAGS="-frtti -std=c++11 $GRPC_PYTHON_CFLAGS"
-fi
 
 ############################
 # Perform build operations #