Merge pull request #5125 from jtattermusch/upport_kokoro_release_python

Upport kokoro/release/python from 3.6.x branch

kokoro/release/python/linux/build_artifacts.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -ex
+
+# change to repo root
+pushd $(dirname $0)/../../../..
+
+export REPO_DIR=protobuf
+export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
+export BUILD_COMMIT=v$BUILD_VERSION
+export PLAT=x86_64
+export UNICODE_WIDTH=32
+export MACOSX_DEPLOYMENT_TARGET=10.9
+
+mkdir artifacts
+export ARTIFACT_DIR=$(pwd)/artifacts
+
+git clone https://github.com/matthew-brett/multibuild.git
+cp kokoro/release/python/linux/config.sh config.sh
+
+build_artifact_version() {
+  MB_PYTHON_VERSION=$1
+
+  # Clean up env
+  rm -rf venv
+  sudo rm -rf protobuf
+  git clone https://github.com/google/protobuf.git
+
+  source multibuild/common_utils.sh
+  source multibuild/travis_steps.sh
+  before_install
+
+  clean_code $REPO_DIR $BUILD_COMMIT
+  sed -i '/Wno-sign-compare/a \ \ \ \ \ \ \ \ extra_compile_args.append("-std=c++11")' $REPO_DIR/python/setup.py
+  cat $REPO_DIR/python/setup.py
+
+  build_wheel $REPO_DIR/python $PLAT
+
+  mv wheelhouse/* $ARTIFACT_DIR
+}
+
+build_artifact_version 2.7
+build_artifact_version 3.4
+build_artifact_version 3.5
+build_artifact_version 3.6

kokoro/release/python/linux/config.sh

@@ -0,0 +1,48 @@
+# Define custom utilities
+# Test for OSX with [ -n "$IS_OSX" ]
+
+function pre_build {
+    # Any stuff that you need to do before you start building the wheels
+    # Runs in the root directory of this repository.
+    pushd protobuf
+
+    # Build protoc
+    ./autogen.sh
+    ./configure
+
+    CXXFLAGS="-fPIC -g -O2" ./configure
+    make -j8
+
+    # Generate python dependencies.
+    pushd python
+    python setup.py build_py
+    popd
+
+    popd
+}
+
+function bdist_wheel_cmd {
+    # Builds wheel with bdist_wheel, puts into wheelhouse
+    #
+    # It may sometimes be useful to use bdist_wheel for the wheel building
+    # process.  For example, versioneer has problems with versions which are
+    # fixed with bdist_wheel:
+    # https://github.com/warner/python-versioneer/issues/121
+    local abs_wheelhouse=$1
+
+    # Modify build version
+    pwd
+    ls
+    python setup.py bdist_wheel --cpp_implementation --compile_static_extension
+    cp dist/*.whl $abs_wheelhouse
+}
+
+function build_wheel {
+    build_wheel_cmd "bdist_wheel_cmd" $@
+}
+
+function run_tests {
+    # Runs tests on installed distribution from an empty directory
+    python --version
+    python -c "from google.protobuf.pyext import _message;"
+}

kokoro/release/python/linux/release.cfg

@@ -0,0 +1,8 @@
+# Configuration for Linux release builds
+build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}

kokoro/release/python/macos/build_artifacts.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -ex
+
+# change to repo root
+pushd $(dirname $0)/../../../..
+
+export REPO_DIR=protobuf
+export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
+export BUILD_COMMIT=v$BUILD_VERSION
+export PLAT=x86_64
+export UNICODE_WIDTH=32
+export MACOSX_DEPLOYMENT_TARGET=10.9
+export TRAVIS_OS_NAME="osx"
+
+mkdir artifacts
+export ARTIFACT_DIR=$(pwd)/artifacts
+
+git clone https://github.com/matthew-brett/multibuild.git
+cp kokoro/release/python/linux/config.sh config.sh
+
+OLD_PATH=$PATH
+
+build_artifact_version() {
+  MB_PYTHON_VERSION=$1
+
+  # Clean up env
+  rm -rf venv
+  sudo rm -rf protobuf
+  git clone https://github.com/google/protobuf.git
+  export PATH=$OLD_PATH
+
+  source multibuild/common_utils.sh
+  source multibuild/travis_steps.sh
+  before_install
+
+  clean_code $REPO_DIR $BUILD_COMMIT
+
+  build_wheel $REPO_DIR/python $PLAT
+
+  mv wheelhouse/* $ARTIFACT_DIR
+}
+
+build_artifact_version 2.7
+build_artifact_version 3.4
+build_artifact_version 3.5
+build_artifact_version 3.6

kokoro/release/python/macos/release.cfg

@@ -0,0 +1,8 @@
+# Configuration for Mac OSX release builds
+build_file: "protobuf/kokoro/release/python/macos/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}

kokoro/release/python/windows/build_artifacts.bat

@@ -0,0 +1,53 @@
+REM Move scripts to root
+cd github\protobuf
+copy kokoro\release\python\windows\build_single_artifact.bat build_single_artifact.bat
+
+REM Set environment variables
+set REPO_DIR=protobuf
+set PACKAGE_NAME=protobuf
+set BUILD_DLL=OFF
+set UNICODE=ON
+set PB_TEST_DEP="six==1.9"
+set OTHER_TEST_DEP="setuptools==38.5.1"
+set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH%
+for /f "tokens=*" %%i in ( 'grep -i "version" python/google/protobuf/__init__.py ^| grep -o "'.*'"' ) do set BUILD_VERSION=%%i
+set BUILD_COMMIT=v%BUILD_VERSION:'=%
+
+REM Fetch multibuild
+git clone https://github.com/matthew-brett/multibuild.git
+
+REM Install zlib
+mkdir zlib
+curl -L -o zlib.zip http://www.winimage.com/zLibDll/zlib123dll.zip
+curl -L -o zlib-src.zip http://www.winimage.com/zLibDll/zlib123.zip
+7z x zlib.zip -ozlib
+7z x zlib-src.zip -ozlib\include
+SET ZLIB_ROOT=%cd%\zlib
+del /Q zlib.zip
+del /Q zlib-src.zip
+
+REM Create directory for artifacts
+SET ARTIFACT_DIR=%cd%\artifacts
+mkdir %ARTIFACT_DIR%
+
+REM Build wheel
+
+SET PYTHON=C:\python35_32bit
+SET PYTHON_VERSION=3.5
+SET PYTHON_ARCH=32
+CALL build_single_artifact.bat
+
+SET PYTHON=C:\python35
+SET PYTHON_VERSION=3.5
+SET PYTHON_ARCH=64
+CALL build_single_artifact.bat
+
+SET PYTHON=C:\python36_32bit
+SET PYTHON_VERSION=3.6
+SET PYTHON_ARCH=32
+CALL build_single_artifact.bat
+
+SET PYTHON=C:\python36
+SET PYTHON_VERSION=3.6
+SET PYTHON_ARCH=64
+CALL build_single_artifact.bat

kokoro/release/python/windows/build_single_artifact.bat

@@ -0,0 +1,62 @@
+setlocal
+
+if %PYTHON%==C:\python35_32bit set generator=Visual Studio 14
+if %PYTHON%==C:\python35_32bit set vcplatform=Win32
+
+if %PYTHON%==C:\python35 set generator=Visual Studio 14 Win64
+if %PYTHON%==C:\python35 set vcplatform=x64
+
+if %PYTHON%==C:\python36_32bit set generator=Visual Studio 14
+if %PYTHON%==C:\python36_32bit set vcplatform=Win32
+
+if %PYTHON%==C:\python36 set generator=Visual Studio 14 Win64
+if %PYTHON%==C:\python36 set vcplatform=x64
+
+REM Prepend newly installed Python to the PATH of this build (this cannot be
+REM done from inside the powershell script as it would require to restart
+REM the parent CMD process).
+SET PATH=%PYTHON%;%PYTHON%\Scripts;%OLD_PATH%
+python -m pip install -U pip
+pip install wheel
+
+REM Check that we have the expected version and architecture for Python
+python --version
+python -c "import struct; print(struct.calcsize('P') * 8)"
+
+rmdir /s/q protobuf
+git clone https://github.com/google/protobuf.git
+
+REM Checkout release commit
+cd %REPO_DIR%
+git checkout %BUILD_COMMIT%
+
+REM ======================
+REM Build Protobuf Library
+REM ======================
+
+mkdir src\.libs
+
+mkdir vcprojects
+pushd vcprojects
+cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=OFF ../cmake
+msbuild protobuf.sln /p:Platform=%vcplatform% /p:Configuration=Release
+dir /s /b
+popd
+copy vcprojects\Release\libprotobuf.lib src\.libs\libprotobuf.a
+copy vcprojects\Release\libprotobuf-lite.lib src\.libs\libprotobuf-lite.a
+SET PATH=%cd%\vcprojects\Release;%PATH%
+dir vcprojects\Release
+
+REM ======================
+REM Build python library
+REM ======================
+
+cd python
+
+REM sed -i 's/\ extra_compile_args\ =\ \[\]/\ extra_compile_args\ =\ \[\'\/MT\'\]/g' setup.py
+
+python setup.py bdist_wheel --cpp_implementation --compile_static_extension
+dir dist
+copy dist\* %ARTIFACT_DIR%
+dir %ARTIFACT_DIR%
+cd ..\..

kokoro/release/python/windows/release.cfg

@@ -0,0 +1,8 @@
+# Configuration for Windows protoc release builds
+build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}