diff --git a/protoc-artifacts/Dockerfile b/protoc-artifacts/Dockerfile deleted file mode 100644 index 278bc7452d..0000000000 --- a/protoc-artifacts/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -FROM centos:6.9 - -RUN yum install -y git \ - tar \ - wget \ - make \ - autoconf \ - curl-devel \ - unzip \ - automake \ - libtool \ - glibc-static.i686 \ - glibc-devel \ - glibc-devel.i686 \ - && \ - yum clean all - -# Install Java 8 -RUN wget -q --no-cookies --no-check-certificate \ - --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \ - "http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz" \ - -O - | tar xz -C /var/local -ENV JAVA_HOME /var/local/jdk1.8.0_131 -ENV PATH $JAVA_HOME/bin:$PATH - -# Install Maven -RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -O - | \ - tar xz -C /var/local -ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH - -# Install GCC 4.8 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d && \ - bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' && \ - bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" && \ - sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo && \ - rpm --rebuilddb - -RUN yum install -y devtoolset-2-gcc \ - devtoolset-2-gcc-c++ \ - devtoolset-2-binutils \ - devtoolset-2-libstdc++-devel \ - devtoolset-2-libatomic-devel \ - libatomic \ - devtoolset-2-gcc.i686 \ - devtoolset-2-gcc-c++.i686 \ - devtoolset-2-binutils.i686 \ - devtoolset-2-libstdc++-devel.i686 \ - devtoolset-2-libatomic-devel.i686 \ - libatomic.i686 && \ - yum clean all - -COPY scl-enable-devtoolset.sh /var/local/ - -# Start in devtoolset environment that uses GCC 4.7 -ENTRYPOINT ["/var/local/scl-enable-devtoolset.sh"] diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md index 8fc366935d..496be7784c 100644 --- a/protoc-artifacts/README.md +++ b/protoc-artifacts/README.md @@ -2,20 +2,13 @@ ``protoc`` is the compiler for ``.proto`` files. It generates language bindings for the messages and/or RPC services from ``.proto`` files. -Because ``protoc`` is a native executable, the scripts under this directory -build and publish a ``protoc`` executable (a.k.a. artifact) to Maven -repositories. The artifact can be used by build automation tools so that users -would not need to compile and install ``protoc`` for their systems. +Because ``protoc`` is a native executable, we build and publish a ``protoc`` +executable (a.k.a. artifact) to Maven repositories. The artifact can be used by +build automation tools so that users would not need to compile and install +``protoc`` for their systems. -If you would like us to publish protoc artifact for a new platform, please send -us a pull request to add support for the new platform. You would need to change -the following files: - -* [build-protoc.sh](build-protoc.sh): script to cross-build the protoc for your - platform. -* [pom.xml](pom.xml): script to upload artifacts to maven. -* [build-zip.sh](build-zip.sh): script to package published maven artifacts in - our release page. +The [pom.xml](pom.xml) file specifies configuration details used by Maven to +publish the protoc binaries. This is only used internally for releases. ## Maven Location The published protoc artifacts are available on Maven here: @@ -32,130 +25,3 @@ The name of a published ``protoc`` artifact is in the following format: Note that artifacts for linux/macos also have the `.exe` suffix but they are not windows binaries. - -## System requirement -Install [Apache Maven](http://maven.apache.org/) if you don't have it. - -The scripts only work under Unix-like environments, e.g., Linux, MacOSX, and -Cygwin or MinGW for Windows. Please see ``README.md`` of the Protobuf project -for how to set up the build environment. - -## Building from a freshly checked-out source - -If you just checked out the Protobuf source from github, you need to -generate the configure script. - -Under the protobuf project directory: - - -``` -$ ./autogen.sh -``` - -### Build the artifact for each platform - -Run the build-protoc.sh script under this protoc-artifacts directory to build the protoc -artifact for each platform. For example: - -``` -$ cd protoc-artifacts -$ ./build-protoc.sh linux x86_64 protoc -``` - -The above command will produce a `target/linux/x86_64/protoc` binary under the -protoc-artifacts directory. - -For a list of supported platforms, see the comments in the build-protoc.sh -script. We only use this script to build artifacts on Ubuntu and MacOS (both -with x86_64, and do cross-compilation for other platforms. - -### Tips for building for Linux -We build on Centos 6.9 to provide a good compatibility for not very new -systems. We have provided a ``Dockerfile`` under this directory to build the -environment. It has been tested with Docker 1.6.1. - -To build a image: - -``` -$ docker build -t protoc-artifacts . -``` - -To run the image: - -``` -$ docker run -it --rm=true protoc-artifacts bash -``` - -To checkout protobuf (run within the container): - -``` -$ # Replace v3.5.1 with the version you want -$ wget -O - https://github.com/protocolbuffers/protobuf/archive/v3.5.1.tar.gz | tar xvzp -``` - -### Windows build -We no longer use scripts in this directory to build windows artifacts. Instead, -we use Visual Studio 2015 to build our windows release artifacts. See our -[kokoro windows build scripts here](../kokoro/release/protoc/windows/build.bat). - -To upload windows artifacts, copy the built binaries into this directory and -put it into the target/windows/(x86_64|x86_32) directory the same way as the -artifacts for other platforms. That will allow the maven script to find and -upload the artifacts to maven. - -## To push artifacts to Maven Central -Before you can upload artifacts to Maven Central repository, make sure you have -read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to -configure GPG and Sonatype account. - -Before you do the deployment, make sure you have built the protoc artifacts for -every supported platform and put them under the target directory. Example -target directory layout: - - + pom.xml - + target - + linux - + x86_64 - protoc.exe - + x86_32 - protoc.exe - + aarch_64 - protoc.exe - + ppcle_64 - protoc.exe - + s390_64 - protoc.exe - + osx - + x86_64 - protoc.exe - + x86_32 - protoc.exe - + windows - + x86_64 - protoc.exe - + x86_32 - protoc.exe - -You will need to build the artifacts on multiple machines and gather them -together into one place. - -Use the following command to deploy artifacts for the host platform to a -staging repository. - -``` -$ mvn deploy -P release -``` - -It creates a new staging repository. Go to -https://oss.sonatype.org/#stagingRepositories and find the repository, usually -in the name like ``comgoogle-123``. Verify that the staging repository has all -the binaries, close and release this repository. - - -## Tested build environments -We have successfully built artifacts on the following environments: -- Linux x86_32 and x86_64: - - Centos 6.9 (within Docker 1.6.1) - - Ubuntu 14.04.5 64-bit -- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit -- Mac OS X x86_32 and x86_64: Mac OS X 10.9.5 diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh deleted file mode 100755 index 4602842622..0000000000 --- a/protoc-artifacts/build-protoc.sh +++ /dev/null @@ -1,294 +0,0 @@ -#!/bin/bash - -# Builds protoc executable into target///protoc.exe; optionally builds -# protoc plugins into target///protoc-gen-*.exe -# -# Usage: ./build-protoc.sh -# -# can be "protoc" or "protoc-gen-javalite". Supported -# combinations: -# HOST -# cygwin windows x86_32 Requires: i686-w64-mingw32-gcc -# cygwin windows x86_64 Requires: x86_64-w64-mingw32-gcc -# linux linux aarch_64 Requires: g++-aarch64-linux-gnu -# linux linux x86_32 -# linux linux x86_64 -# linux windows x86_32 Requires: i686-w64-mingw32-gcc -# linux windows x86_64 Requires: x86_64-w64-mingw32-gcc -# macos osx x86_32 -# macos osx x86_64 -# mingw windows x86_32 -# mingw windows x86_64 -# -# Before running this script, make sure you have generated the configure script -# in the parent directory (i.e., run ./autogen.sh there). - -OS=$1 -ARCH=$2 -MAKE_TARGET=$3 - -if [[ $# < 3 ]]; then - echo "Not enough arguments provided." - exit 1 -fi - -case $MAKE_TARGET in - protoc-gen-javalite) - ;; - protoc) - ;; - *) - echo "Target ""$MAKE_TARGET"" invalid." - exit 1 -esac - -# Under Cygwin, bash doesn't have these in PATH when called from Maven which -# runs in Windows version of Java. -export PATH="/bin:/usr/bin:$PATH" - -############################################################################ -# Helper functions -############################################################################ -E_PARAM_ERR=98 -E_ASSERT_FAILED=99 - -# Usage: -fail() -{ - echo "ERROR: $1" - echo - exit $E_ASSERT_FAILED -} - -# Usage: assertEq VAL1 VAL2 $LINENO -assertEq () -{ - lineno=$3 - if [ -z "$lineno" ]; then - echo "lineno not given" - exit $E_PARAM_ERR - fi - - if [[ "$1" != "$2" ]]; then - echo "Assertion failed: \"$1\" == \"$2\"" - echo "File \"$0\", line $lineno" # Give name of file and line number. - exit $E_ASSERT_FAILED - fi -} - -# Checks the artifact is for the expected architecture -# Usage: checkArch -checkArch () -{ - echo - echo "Checking file format ..." - if [[ "$OS" == windows || "$OS" == linux ]]; then - format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")" - echo Format=$format - if [[ "$OS" == linux ]]; then - host_machine="$(uname -m)"; - if [[ "$ARCH" == x86_32 ]]; then - assertEq $format "elf32-i386" $LINENO - elif [[ "$ARCH" == x86_64 ]]; then - assertEq $format "elf64-x86-64" $LINENO - elif [[ "$ARCH" == aarch_64 ]]; then - assertEq $format "elf64-little" $LINENO - elif [[ "$ARCH" == s390_64 ]]; then - if [[ $host_machine == s390x ]];then - assertEq $format "elf64-s390" $LINENO - else - assertEq $format "elf64-big" $LINENO - fi - elif [[ "$ARCH" == ppcle_64 ]]; then - if [[ $host_machine == ppc64le ]];then - assertEq $format "elf64-powerpcle" $LINENO - else - assertEq $format "elf64-little" $LINENO - fi - else - fail "Unsupported arch: $ARCH" - fi - else - # $OS == windows - if [[ "$ARCH" == x86_32 ]]; then - assertEq $format "pei-i386" $LINENO - elif [[ "$ARCH" == x86_64 ]]; then - assertEq $format "pei-x86-64" $LINENO - else - fail "Unsupported arch: $ARCH" - fi - fi - elif [[ "$OS" == osx ]]; then - format="$(file -b "$1" | grep -o "[^ ]*$")" - echo Format=$format - if [[ "$ARCH" == x86_32 ]]; then - assertEq $format "i386" $LINENO - elif [[ "$ARCH" == x86_64 ]]; then - assertEq $format "x86_64" $LINENO - else - fail "Unsupported arch: $ARCH" - fi - else - fail "Unsupported system: $OS" - fi - echo -} - -# Checks the dependencies of the artifact. Artifacts should only depend on -# system libraries. -# Usage: checkDependencies -checkDependencies () -{ - if [[ "$OS" == windows ]]; then - dump_cmd='objdump -x '"$1"' | fgrep "DLL Name"' - white_list="KERNEL32\.dll\|msvcrt\.dll" - elif [[ "$OS" == linux ]]; then - host_machine="$(uname -m)"; - dump_cmd='ldd '"$1" - if [[ "$ARCH" == x86_32 ]]; then - white_list="linux-gate\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux\.so\.2" - elif [[ "$ARCH" == x86_64 ]]; then - white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-x86-64\.so\.2" - elif [[ "$ARCH" == s390_64 ]]; then - if [[ $host_machine != s390x ]];then - dump_cmd='objdump -p '"$1"' | grep NEEDED' - fi - white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.1" - elif [[ "$ARCH" == ppcle_64 ]]; then - if [[ $host_machine != ppc64le ]];then - dump_cmd='objdump -p '"$1"' | grep NEEDED' - fi - white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2" - elif [[ "$ARCH" == aarch_64 ]]; then - dump_cmd='objdump -p '"$1"' | grep NEEDED' - white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1" - fi - elif [[ "$OS" == osx ]]; then - dump_cmd='otool -L '"$1"' | fgrep dylib' - white_list="libz\.1\.dylib\|libstdc++\.6\.dylib\|libSystem\.B\.dylib" - fi - if [[ -z "$white_list" || -z "$dump_cmd" ]]; then - fail "Unsupported platform $OS-$ARCH." - fi - echo "Checking for expected dependencies ..." - eval $dump_cmd | grep -i "$white_list" || fail "doesn't show any expected dependencies" - echo "Checking for unexpected dependencies ..." - eval $dump_cmd | grep -i -v "$white_list" - ret=$? - if [[ $ret == 0 ]]; then - fail "found unexpected dependencies (listed above)." - elif [[ $ret != 1 ]]; then - fail "Error when checking dependencies." - fi # grep returns 1 when "not found", which is what we expect - echo "Dependencies look good." - echo -} -############################################################################ - -echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$MAKE_TARGET" - -CONFIGURE_ARGS="--disable-shared" - -if [[ "$OS" == windows ]]; then - MAKE_TARGET="${MAKE_TARGET}.exe" -fi - -# Override the default value set in configure.ac that has '-g' which produces -# huge binary. -CXXFLAGS="-DNDEBUG" -LDFLAGS="" - -if [[ "$(uname)" == CYGWIN* ]]; then - assertEq "$OS" windows $LINENO - # Use mingw32 compilers because executables produced by Cygwin compiler - # always have dependency on Cygwin DLL. - if [[ "$ARCH" == x86_64 ]]; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" - elif [[ "$ARCH" == x86_32 ]]; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-pc-mingw32" - else - fail "Unsupported arch by CYGWIN: $ARCH" - fi -elif [[ "$(uname)" == MINGW32* ]]; then - assertEq "$OS" windows $LINENO - assertEq "$ARCH" x86_32 $LINENO -elif [[ "$(uname)" == MINGW64* ]]; then - assertEq "$OS" windows $LINENO - assertEq "$ARCH" x86_64 $LINENO -elif [[ "$(uname)" == Linux* ]]; then - if [[ "$OS" == linux ]]; then - if [[ "$ARCH" == x86_64 ]]; then - CXXFLAGS="$CXXFLAGS -m64" - elif [[ "$ARCH" == x86_32 ]]; then - CXXFLAGS="$CXXFLAGS -m32" - elif [[ "$ARCH" == aarch_64 ]]; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=aarch64-linux-gnu" - elif [[ "$ARCH" == ppcle_64 ]]; then - CXXFLAGS="$CXXFLAGS -m64" - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=powerpc64le-linux-gnu" - elif [[ "$ARCH" == s390_64 ]]; then - CXXFLAGS="$CXXFLAGS -m64" - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=s390x-linux-gnu" - else - fail "Unsupported arch: $ARCH" - fi - elif [[ "$OS" == windows ]]; then - # Cross-compilation for Windows - CONFIGURE_ARGS="$CONFIGURE_ARGS" - if [[ "$ARCH" == x86_64 ]]; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" - elif [[ "$ARCH" == x86_32 ]]; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-w64-mingw32" - else - fail "Unsupported arch: $ARCH" - fi - else - fail "Cannot build $OS on $(uname)" - fi -elif [[ "$(uname)" == Darwin* ]]; then - assertEq "$OS" osx $LINENO - # Make the binary compatible with OSX 10.7 and later - CXXFLAGS="$CXXFLAGS -mmacosx-version-min=10.7" - if [[ "$ARCH" == x86_64 ]]; then - CXXFLAGS="$CXXFLAGS -m64" - elif [[ "$ARCH" == x86_32 ]]; then - CXXFLAGS="$CXXFLAGS -m32" - else - fail "Unsupported arch: $ARCH" - fi -else - fail "Unsupported system: $(uname)" -fi - -# Statically link libgcc and libstdc++. -# -s to produce stripped binary. -if [[ "$OS" == windows ]]; then - # Also static link libpthread required by mingw64 - LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -s" -elif [[ "$OS" != osx ]]; then - # And they don't work under Mac. - LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -s" -fi - -export CXXFLAGS LDFLAGS - -# Nested double quotes are unintuitive, but it works. -cd "$(dirname "$0")" - -WORKING_DIR="$(pwd)" -BUILD_DIR="build/$OS/$ARCH" -TARGET_FILE="target/$OS/$ARCH/$MAKE_TARGET.exe" - -mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR" && - ../../../../configure $CONFIGURE_ARGS && - cd src && make $MAKE_TARGET -j8 && - cd "$WORKING_DIR" && mkdir -p $(dirname $TARGET_FILE) && - cp $BUILD_DIR/src/$MAKE_TARGET $TARGET_FILE || - exit 1 - -if [[ "$OS" == osx ]]; then - # Since Mac linker doesn't accept "-s", we need to run strip - strip $TARGET_FILE || exit 1 -fi - -checkArch $TARGET_FILE && checkDependencies $TARGET_FILE diff --git a/protoc-artifacts/build-zip.sh b/protoc-artifacts/build-zip.sh deleted file mode 100755 index 1d977259f3..0000000000 --- a/protoc-artifacts/build-zip.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - cat < - -TARGET: protoc | protoc-gen-javalite - -Example: - $ $0 protoc 3.0.0 - $ $0 protoc-gen-javalite 3.0.0 - -This script will download pre-built protoc or protoc plugin binaries from maven -repository and create .zip packages suitable to be included in the github -release page. If the target is protoc, well-known type .proto files will also be -included. Each invocation will create 8 zip packages: - dist/--win32.zip - dist/--win64.zip - dist/--osx-aarch_64.zip - dist/--osx-x86_64.zip - dist/--linux-x86_32.zip - dist/--linux-x86_64.zip - dist/--linux-aarch_64.zip - dist/--linux-ppcle_64.zip - dist/--linux-s390_64.zip -EOF - exit 1 -fi - -TARGET=$1 -VERSION_NUMBER=$2 - -# pairs. -declare -a FILE_NAMES=( \ - win32.zip windows-x86_32.exe \ - win64.zip windows-x86_64.exe \ - osx-aarch_64.zip osx-aarch_64.exe \ - osx-x86_64.zip osx-x86_64.exe \ - linux-x86_32.zip linux-x86_32.exe \ - linux-x86_64.zip linux-x86_64.exe \ - linux-aarch_64.zip linux-aarch_64.exe \ - linux-ppcle_64.zip linux-ppcle_64.exe \ - linux-s390_64.zip linux-s390_64.exe \ -) - -# List of all well-known types to be included. -declare -a WELL_KNOWN_TYPES=( \ - google/protobuf/descriptor.proto \ - google/protobuf/any.proto \ - google/protobuf/api.proto \ - google/protobuf/duration.proto \ - google/protobuf/empty.proto \ - google/protobuf/field_mask.proto \ - google/protobuf/source_context.proto \ - google/protobuf/struct.proto \ - google/protobuf/timestamp.proto \ - google/protobuf/type.proto \ - google/protobuf/wrappers.proto \ - google/protobuf/compiler/plugin.proto \ -) - -set -e - -# A temporary working directory to put all files. -DIR=$(mktemp -d) - -# Copy over well-known types. -mkdir -p ${DIR}/include/google/protobuf/compiler -for PROTO in ${WELL_KNOWN_TYPES[@]}; do - cp -f ../src/${PROTO} ${DIR}/include/${PROTO} -done - -# Create a readme file. -cat < ${DIR}/readme.txt -Protocol Buffers - Google's data interchange format -Copyright 2008 Google Inc. -https://developers.google.com/protocol-buffers/ - -This package contains a precompiled binary version of the protocol buffer -compiler (protoc). This binary is intended for users who want to use Protocol -Buffers in languages other than C++ but do not want to compile protoc -themselves. To install, simply place this binary somewhere in your PATH. - -If you intend to use the included well known types then don't forget to -copy the contents of the 'include' directory somewhere as well, for example -into '/usr/local/include/'. - -Please refer to our official github site for more installation instructions: - https://github.com/protocolbuffers/protobuf -EOF - -mkdir -p dist -mkdir -p ${DIR}/bin -# Create a zip file for each binary. -for((i=0;i<${#FILE_NAMES[@]};i+=2));do - ZIP_NAME=${FILE_NAMES[$i]} - if [ ${ZIP_NAME:0:3} = "win" ]; then - BINARY="$TARGET.exe" - else - BINARY="$TARGET" - fi - BINARY_NAME=${FILE_NAMES[$(($i+1))]} - BINARY_URL=https://repo1.maven.org/maven2/com/google/protobuf/$TARGET/${VERSION_NUMBER}/$TARGET-${VERSION_NUMBER}-${BINARY_NAME} - if ! wget ${BINARY_URL} -O ${DIR}/bin/$BINARY &> /dev/null; then - echo "[ERROR] Failed to download ${BINARY_URL}" >&2 - echo "[ERROR] Skipped $TARGET-${VERSION_NAME}-${ZIP_NAME}" >&2 - continue - fi - TARGET_ZIP_FILE=`pwd`/dist/$TARGET-${VERSION_NUMBER}-${ZIP_NAME} - pushd $DIR &> /dev/null - chmod +x bin/$BINARY - if [ "$TARGET" = "protoc" ]; then - zip -r ${TARGET_ZIP_FILE} include bin readme.txt &> /dev/null - else - zip -r ${TARGET_ZIP_FILE} bin &> /dev/null - fi - rm bin/$BINARY - popd &> /dev/null - echo "[INFO] Successfully created ${TARGET_ZIP_FILE}" -done