Merge pull request #23483 from emkornfield/fg

Adds option of using ASM for BoringSSL in Python.
pull/23553/head
Lidi Zheng 4 years ago committed by GitHub
commit 218a2ff95f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 35
      setup.py
  2. 8
      src/boringssl/gen_build_yaml.py
  3. 169
      src/python/grpcio/grpc_core_dependencies.py
  4. 13
      templates/src/python/grpcio/grpc_core_dependencies.py.template
  5. 1
      tools/buildgen/plugins/check_attrs.py
  6. 1
      tools/run_tests/artifacts/build_artifact_python.sh

@ -17,6 +17,12 @@
# undesirable behaviors or errors. # undesirable behaviors or errors.
import setuptools import setuptools
# Monkey Patch the unix compiler to accept ASM
# files used by boring SSL.
from distutils.unixccompiler import UnixCCompiler
UnixCCompiler.src_extensions.append('.S')
del UnixCCompiler
from distutils import cygwinccompiler from distutils import cygwinccompiler
from distutils import extension as _extension from distutils import extension as _extension
from distutils import util from distutils import util
@ -97,6 +103,9 @@ CLASSIFIERS = [
'License :: OSI Approved :: Apache Software License', 'License :: OSI Approved :: Apache Software License',
] ]
BUILD_WITH_BORING_SSL_ASM = os.environ.get('GRPC_BUILD_WITH_BORING_SSL_ASM',
True)
# Environment variable to determine whether or not the Cython extension should # Environment variable to determine whether or not the Cython extension should
# *use* Cython or use the generated C files. Note that this requires the C files # *use* Cython or use the generated C files. Note that this requires the C files
# to have been generated by building first *with* Cython support. Even if this # to have been generated by building first *with* Cython support. Even if this
@ -261,9 +270,30 @@ if BUILD_WITH_SYSTEM_ZLIB:
if BUILD_WITH_SYSTEM_CARES: if BUILD_WITH_SYSTEM_CARES:
EXTENSION_LIBRARIES += ('cares',) EXTENSION_LIBRARIES += ('cares',)
DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600)) DEFINE_MACROS = (('_WIN32_WINNT', 0x600),)
asm_files = []
asm_key = ''
if BUILD_WITH_BORING_SSL_ASM:
LINUX_X86_64 = 'linux-x86_64'
LINUX_ARM = 'linux-arm'
if LINUX_X86_64 == util.get_platform():
asm_key = 'crypto_linux_x86_64'
elif LINUX_ARM == util.get_platform():
asm_key = 'crypto_linux_arm'
elif "mac" in util.get_platform() and "x86_64" in util.get_platform():
asm_key = 'crypto_mac_x86_64'
else:
print("ASM Builds for BoringSSL currently not supported on:",
util.get_platform())
if asm_key:
asm_files = grpc_core_dependencies.ASM_SOURCE_FILES[asm_key]
else:
DEFINE_MACROS += (('OPENSSL_NO_ASM', 1),)
if not DISABLE_LIBC_COMPATIBILITY: if not DISABLE_LIBC_COMPATIBILITY:
DEFINE_MACROS += (('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) DEFINE_MACROS += (('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
if "win32" in sys.platform: if "win32" in sys.platform:
# TODO(zyc): Re-enable c-ares on x64 and x86 windows after fixing the # TODO(zyc): Re-enable c-ares on x64 and x86 windows after fixing the
# ares_library_init compilation issue # ares_library_init compilation issue
@ -328,7 +358,8 @@ def cython_extensions_and_necessity():
extensions = [ extensions = [
_extension.Extension( _extension.Extension(
name=module_name, name=module_name,
sources=[module_file] + list(CYTHON_HELPER_C_FILES) + core_c_files, sources=([module_file] + list(CYTHON_HELPER_C_FILES) +
core_c_files + asm_files),
include_dirs=list(EXTENSION_INCLUDE_DIRECTORIES), include_dirs=list(EXTENSION_INCLUDE_DIRECTORIES),
libraries=list(EXTENSION_LIBRARIES), libraries=list(EXTENSION_LIBRARIES),
define_macros=list(DEFINE_MACROS), define_macros=list(DEFINE_MACROS),

@ -46,6 +46,10 @@ class Grpc(object):
def WriteFiles(self, files): def WriteFiles(self, files):
test_binaries = ['ssl_test', 'crypto_test'] test_binaries = ['ssl_test', 'crypto_test']
asm_outputs = {
key: value for key, value in files.items() if any(
f.endswith(".S") or f.endswith(".asm") for f in value)
}
self.yaml = { self.yaml = {
'#': '#':
'generated with src/boringssl/gen_build_yaml.py', 'generated with src/boringssl/gen_build_yaml.py',
@ -65,6 +69,10 @@ class Grpc(object):
'src': 'src':
sorted( sorted(
map_dir(f) for f in files['ssl'] + files['crypto']), map_dir(f) for f in files['ssl'] + files['crypto']),
'asm_src': {
k: [map_dir(f) for f in value
] for k, value in asm_outputs.items()
},
'headers': 'headers':
sorted( sorted(
map_dir(f) map_dir(f)

@ -911,3 +911,172 @@ CORE_SOURCE_FILES = [
'third_party/zlib/uncompr.c', 'third_party/zlib/uncompr.c',
'third_party/zlib/zutil.c', 'third_party/zlib/zutil.c',
] ]
ASM_SOURCE_FILES = {
'crypto_ios_aarch64': [
'third_party/boringssl-with-bazel/ios-aarch64/crypto/chacha/chacha-armv8.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/test/trampoline-armv8.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/aesv8-armx64.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/armv8-mont.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/ghashv8-armx64.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/ghash-neon-armv8.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/sha1-armv8.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/sha256-armv8.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/sha512-armv8.S',
'third_party/boringssl-with-bazel/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S',
],
'crypto_mac_x86_64': [
'third_party/boringssl-with-bazel/mac-x86_64/crypto/chacha/chacha-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/cipher_extra/aes128gcmsiv-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/cipher_extra/chacha20_poly1305_x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/test/trampoline-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/aesni-gcm-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/aesni-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/ghash-ssse3-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/ghash-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/md5-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/p256-x86_64-asm.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/p256_beeu-x86_64-asm.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/rdrand-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/rsaz-avx2.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/sha1-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/sha256-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/sha512-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/vpaes-x86_64.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/x86_64-mont5.S',
'third_party/boringssl-with-bazel/mac-x86_64/crypto/fipsmodule/x86_64-mont.S',
],
'crypto_win_x86': [
'third_party/boringssl-with-bazel/win-x86/crypto/chacha/chacha-x86.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/test/trampoline-x86.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/aesni-x86.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/bn-586.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/co-586.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/ghash-ssse3-x86.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/ghash-x86.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/md5-586.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/sha1-586.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/sha256-586.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/sha512-586.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/vpaes-x86.asm',
'third_party/boringssl-with-bazel/win-x86/crypto/fipsmodule/x86-mont.asm',
],
'crypto_linux_ppc64le': [
'third_party/boringssl-with-bazel/linux-ppc64le/crypto/test/trampoline-ppc.S',
'third_party/boringssl-with-bazel/linux-ppc64le/crypto/fipsmodule/aesp8-ppc.S',
'third_party/boringssl-with-bazel/linux-ppc64le/crypto/fipsmodule/ghashp8-ppc.S',
],
'crypto_mac_x86': [
'third_party/boringssl-with-bazel/mac-x86/crypto/chacha/chacha-x86.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/test/trampoline-x86.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/aesni-x86.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/bn-586.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/co-586.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/ghash-ssse3-x86.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/ghash-x86.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/md5-586.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/sha1-586.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/sha256-586.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/sha512-586.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/vpaes-x86.S',
'third_party/boringssl-with-bazel/mac-x86/crypto/fipsmodule/x86-mont.S',
],
'crypto_linux_x86': [
'third_party/boringssl-with-bazel/linux-x86/crypto/chacha/chacha-x86.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/test/trampoline-x86.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/aesni-x86.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/bn-586.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/co-586.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/ghash-ssse3-x86.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/ghash-x86.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/md5-586.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/sha1-586.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/sha256-586.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/sha512-586.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/vpaes-x86.S',
'third_party/boringssl-with-bazel/linux-x86/crypto/fipsmodule/x86-mont.S',
],
'crypto_ios_arm': [
'third_party/boringssl-with-bazel/ios-arm/crypto/chacha/chacha-armv4.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/test/trampoline-armv4.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/aesv8-armx32.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/armv4-mont.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/bsaes-armv7.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/ghash-armv4.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/ghashv8-armx32.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/sha1-armv4-large.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/sha256-armv4.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/sha512-armv4.S',
'third_party/boringssl-with-bazel/ios-arm/crypto/fipsmodule/vpaes-armv7.S',
],
'crypto_linux_x86_64': [
'third_party/boringssl-with-bazel/linux-x86_64/crypto/chacha/chacha-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/cipher_extra/aes128gcmsiv-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/cipher_extra/chacha20_poly1305_x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/test/trampoline-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/aesni-gcm-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/aesni-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/ghash-ssse3-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/ghash-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/md5-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/p256-x86_64-asm.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/p256_beeu-x86_64-asm.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/rdrand-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/rsaz-avx2.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/sha1-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/sha256-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/sha512-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/vpaes-x86_64.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/x86_64-mont5.S',
'third_party/boringssl-with-bazel/linux-x86_64/crypto/fipsmodule/x86_64-mont.S',
'third_party/boringssl-with-bazel/src/crypto/hrss/asm/poly_rq_mul.S',
],
'crypto_win_x86_64': [
'third_party/boringssl-with-bazel/win-x86_64/crypto/chacha/chacha-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/cipher_extra/aes128gcmsiv-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/cipher_extra/chacha20_poly1305_x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/test/trampoline-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/aesni-gcm-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/aesni-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/ghash-ssse3-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/ghash-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/md5-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/p256-x86_64-asm.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/p256_beeu-x86_64-asm.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/rdrand-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/rsaz-avx2.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/sha1-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/sha256-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/sha512-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/vpaes-x86_64.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/x86_64-mont5.asm',
'third_party/boringssl-with-bazel/win-x86_64/crypto/fipsmodule/x86_64-mont.asm',
],
'crypto_linux_aarch64': [
'third_party/boringssl-with-bazel/linux-aarch64/crypto/chacha/chacha-armv8.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/test/trampoline-armv8.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/aesv8-armx64.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/armv8-mont.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/ghashv8-armx64.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/ghash-neon-armv8.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/sha1-armv8.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/sha256-armv8.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/sha512-armv8.S',
'third_party/boringssl-with-bazel/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S',
],
'crypto_linux_arm': [
'third_party/boringssl-with-bazel/linux-arm/crypto/chacha/chacha-armv4.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/test/trampoline-armv4.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/aesv8-armx32.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/armv4-mont.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/bsaes-armv7.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/ghash-armv4.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/ghashv8-armx32.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/sha1-armv4-large.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/sha256-armv4.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/sha512-armv4.S',
'third_party/boringssl-with-bazel/linux-arm/crypto/fipsmodule/vpaes-armv7.S',
'third_party/boringssl-with-bazel/src/crypto/curve25519/asm/x25519-asm-arm.S',
'third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_arm_asm.S',
],
}

@ -17,10 +17,13 @@
# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_core_dependencies.py.template`!!! # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_core_dependencies.py.template`!!!
<% <%
srcs = [] srcs = []
asm_srcs = {}
for lib in libs: for lib in libs:
if lib.name in python_dependencies.transitive_deps: if lib.name in python_dependencies.transitive_deps:
for src in lib.src: for src in lib.src:
srcs.append(src) srcs.append(src)
if hasattr(lib, 'asm_src'):
asm_srcs = lib.asm_src
srcs = sorted(set(srcs)) srcs = sorted(set(srcs))
%> %>
CORE_SOURCE_FILES = [ CORE_SOURCE_FILES = [
@ -28,3 +31,13 @@
'${src}', '${src}',
% endfor % endfor
] ]
ASM_SOURCE_FILES = {
% for asm, asm_src in asm_srcs.iteritems():
'${asm}': [
% for src in asm_src:
'${src}',
% endfor
],
% endfor
}

@ -38,6 +38,7 @@ VALID_ATTRIBUTE_KEYS_MAP = {
'uses': anything(), 'uses': anything(),
}, },
'lib': { 'lib': {
'asm_src': anything(),
'baselib': anything(), 'baselib': anything(),
'boringssl': one_of((True,)), 'boringssl': one_of((True,)),
'build_system': anything(), 'build_system': anything(),

@ -53,6 +53,7 @@ clean_non_source_files() {
| grep -v '\.c$' | grep -v '\.cc$' | grep -v '\.cpp$' \ | grep -v '\.c$' | grep -v '\.cc$' | grep -v '\.cpp$' \
| grep -v '\.h$' | grep -v '\.hh$' | grep -v '\.inc$' \ | grep -v '\.h$' | grep -v '\.hh$' | grep -v '\.inc$' \
| grep -v '\.s$' | grep -v '\.py$' | grep -v '\.hpp$' \ | grep -v '\.s$' | grep -v '\.py$' | grep -v '\.hpp$' \
| grep -v '\.S$' | grep -v '\.asm$' \
| while read -r file; do | while read -r file; do
rm -f "$file" || true rm -f "$file" || true
done done

Loading…
Cancel
Save