diff --git a/java/protoc/README.md b/java/protoc/README.md new file mode 100644 index 0000000000..a31ffbe822 --- /dev/null +++ b/java/protoc/README.md @@ -0,0 +1,42 @@ +# Publish pre-compiled protoc artifacts +``protoc`` is the compiler for ``.proto`` files. It generates language bindings +for the messages and/or RPC services from ``.proto`` files. + +Because ``protoc`` is a native executable, the scripts under this directory +publish a ``protoc`` executable (a.k.a. artifact) to Maven repositories. The +artifact can be used by build automation tools so that users would not need to +compile and install ``protoc`` for their systems. + +If you would like us to publish protoc artifact for a new platform, please +open an issue to request it. + +## Maven Location +The published protoc artifacts are available on Maven here: + + https://repo.maven.apache.org/maven2/com/google/protobuf/protoc/ + +## Versioning +The version of the ``protoc`` artifact must be the same as the version of the +Protobuf project. + +## Artifact name +The name of a published ``protoc`` artifact is in the following format: +``protoc---.exe``, e.g., ``protoc-3.6.1-linux-x86_64.exe``. + +Note that artifacts for linux/macos also have the `.exe` suffix but they are +not windows binaries. + +## System requirement +Install [Apache Maven](http://maven.apache.org/) if you don't have it. + +The scripts only work under Unix-like environments, e.g., Linux, MacOSX, and +Cygwin or MinGW for Windows. Please see ``README.md`` of the Protobuf project +for how to set up the build environment. + +## Tested build environments +We have successfully built artifacts on the following environments: +- Linux x86_32 and x86_64: + - Centos 6.9 (within Docker 1.6.1) + - Ubuntu 14.04.5 64-bit +- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit +- Mac OS X x86_32 and x86_64: Mac OS X 10.9.5 diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index b2cd0d5519..9b61b9826a 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.21.3 + 3.21.6 pom Protobuf Compiler diff --git a/kokoro/release/collect_all_artifacts.sh b/kokoro/release/collect_all_artifacts.sh index 6e0d152408..20c6d8d785 100755 --- a/kokoro/release/collect_all_artifacts.sh +++ b/kokoro/release/collect_all_artifacts.sh @@ -40,8 +40,8 @@ mkdir -p protoc/linux_x86 mkdir -p protoc/linux_x64 # Because of maven unrelated reasonse the linux protoc binaries have a dummy .exe extension. # For the Google.Protobuf.Tools nuget, we don't want that exception, so we just remove it. -cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_32/protoc.exe protoc/linux_x86/protoc -cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_64/protoc.exe protoc/linux_x64/protoc +cp ${INPUT_ARTIFACTS_DIR}/kokoro/release/protoc/target/linux/x86_32/protoc.exe protoc/linux_x86/protoc +cp ${INPUT_ARTIFACTS_DIR}/kokoro/release/protoc/target/linux/x86_64/protoc.exe protoc/linux_x64/protoc mkdir -p protoc/macosx_x64 cp ${INPUT_ARTIFACTS_DIR}/build64/src/protoc protoc/macosx_x64/protoc diff --git a/protoc-artifacts/build-protoc.sh b/kokoro/release/protoc/build-protoc.sh similarity index 100% rename from protoc-artifacts/build-protoc.sh rename to kokoro/release/protoc/build-protoc.sh diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh index ca3ff2124a..a9d58b7c75 100755 --- a/kokoro/release/protoc/linux/build.sh +++ b/kokoro/release/protoc/linux/build.sh @@ -18,12 +18,12 @@ git submodule update --init --recursive # directories afterward. -protoc-artifacts/build-protoc.sh linux aarch_64 protoc +kokoro/release/protoc/build-protoc.sh linux aarch_64 protoc -protoc-artifacts/build-protoc.sh linux ppcle_64 protoc +kokoro/release/protoc/build-protoc.sh linux ppcle_64 protoc -protoc-artifacts/build-protoc.sh linux s390_64 protoc +kokoro/release/protoc/build-protoc.sh linux s390_64 protoc -protoc-artifacts/build-protoc.sh linux x86_64 protoc +kokoro/release/protoc/build-protoc.sh linux x86_64 protoc -protoc-artifacts/build-protoc.sh linux x86_32 protoc +kokoro/release/protoc/build-protoc.sh linux x86_32 protoc diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md deleted file mode 100644 index 5511f9aedf..0000000000 --- a/protoc-artifacts/README.md +++ /dev/null @@ -1,161 +0,0 @@ -# Build scripts that publish pre-compiled protoc artifacts -``protoc`` is the compiler for ``.proto`` files. It generates language bindings -for the messages and/or RPC services from ``.proto`` files. - -Because ``protoc`` is a native executable, the scripts under this directory -build and publish a ``protoc`` executable (a.k.a. artifact) to Maven -repositories. The artifact can be used by build automation tools so that users -would not need to compile and install ``protoc`` for their systems. - -If you would like us to publish protoc artifact for a new platform, please send -us a pull request to add support for the new platform. You would need to change -the following files: - -* [build-protoc.sh](build-protoc.sh): script to cross-build the protoc for your - platform. -* [pom.xml](pom.xml): script to upload artifacts to maven. -* [build-zip.sh](build-zip.sh): script to package published maven artifacts in - our release page. - -## Maven Location -The published protoc artifacts are available on Maven here: - - https://repo.maven.apache.org/maven2/com/google/protobuf/protoc/ - -## Versioning -The version of the ``protoc`` artifact must be the same as the version of the -Protobuf project. - -## Artifact name -The name of a published ``protoc`` artifact is in the following format: -``protoc---.exe``, e.g., ``protoc-3.6.1-linux-x86_64.exe``. - -Note that artifacts for linux/macos also have the `.exe` suffix but they are -not windows binaries. - -## System requirement -Install [Apache Maven](http://maven.apache.org/) if you don't have it. - -The scripts only work under Unix-like environments, e.g., Linux, MacOSX, and -Cygwin or MinGW for Windows. Please see ``README.md`` of the Protobuf project -for how to set up the build environment. - -## Building from a freshly checked-out source - -If you just checked out the Protobuf source from github, you need to -pull in any submodules. - -Under the protobuf project directory: - - -``` -git submodule update --init --recursive -``` - -### Build the artifact for each platform - -Run the build-protoc.sh script under this protoc-artifacts directory to build the protoc -artifact for each platform. For example: - -``` -$ cd protoc-artifacts -$ ./build-protoc.sh linux x86_64 protoc -``` - -The above command will produce a `target/linux/x86_64/protoc` binary under the -protoc-artifacts directory. - -For a list of supported platforms, see the comments in the build-protoc.sh -script. We only use this script to build artifacts on Ubuntu and MacOS (both -with x86_64, and do cross-compilation for other platforms. - -### Tips for building for Linux -We build on Centos 6.9 to provide a good compatibility for not very new -systems. We have provided a ``Dockerfile`` under this directory to build the -environment. It has been tested with Docker 1.6.1. - -To build a image: - -``` -$ docker build -t protoc-artifacts . -``` - -To run the image: - -``` -$ docker run -it --rm=true protoc-artifacts bash -``` - -To checkout protobuf (run within the container): - -``` -$ # Replace v3.5.1 with the version you want -$ wget -O - https://github.com/protocolbuffers/protobuf/archive/v3.5.1.tar.gz | tar xvzp -``` - -### Windows build -We no longer use scripts in this directory to build windows artifacts. Instead, -we use Visual Studio 2015 to build our windows release artifacts. See our -[kokoro windows build scripts here](../kokoro/release/protoc/windows/build.bat). - -To upload windows artifacts, copy the built binaries into this directory and -put it into the target/windows/(x86_64|x86_32) directory the same way as the -artifacts for other platforms. That will allow the maven script to find and -upload the artifacts to maven. - -## To push artifacts to Maven Central -Before you can upload artifacts to Maven Central repository, make sure you have -read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to -configure GPG and Sonatype account. - -Before you do the deployment, make sure you have built the protoc artifacts for -every supported platform and put them under the target directory. Example -target directory layout: - - + pom.xml - + target - + linux - + x86_64 - protoc.exe - + x86_32 - protoc.exe - + aarch_64 - protoc.exe - + ppcle_64 - protoc.exe - + s390_64 - protoc.exe - + osx - + x86_64 - protoc.exe - + x86_32 - protoc.exe - + windows - + x86_64 - protoc.exe - + x86_32 - protoc.exe - -You will need to build the artifacts on multiple machines and gather them -together into one place. - -Use the following command to deploy artifacts for the host platform to a -staging repository. - -``` -$ mvn deploy -P release -``` - -It creates a new staging repository. Go to -https://oss.sonatype.org/#stagingRepositories and find the repository, usually -in the name like ``comgoogle-123``. Verify that the staging repository has all -the binaries, close and release this repository. - - -## Tested build environments -We have successfully built artifacts on the following environments: -- Linux x86_32 and x86_64: - - Centos 6.9 (within Docker 1.6.1) - - Ubuntu 14.04.5 64-bit -- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit -- Mac OS X x86_32 and x86_64: Mac OS X 10.9.5 diff --git a/protoc-artifacts/build-zip.sh b/protoc-artifacts/build-zip.sh deleted file mode 100755 index 1d977259f3..0000000000 --- a/protoc-artifacts/build-zip.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - cat < - -TARGET: protoc | protoc-gen-javalite - -Example: - $ $0 protoc 3.0.0 - $ $0 protoc-gen-javalite 3.0.0 - -This script will download pre-built protoc or protoc plugin binaries from maven -repository and create .zip packages suitable to be included in the github -release page. If the target is protoc, well-known type .proto files will also be -included. Each invocation will create 8 zip packages: - dist/--win32.zip - dist/--win64.zip - dist/--osx-aarch_64.zip - dist/--osx-x86_64.zip - dist/--linux-x86_32.zip - dist/--linux-x86_64.zip - dist/--linux-aarch_64.zip - dist/--linux-ppcle_64.zip - dist/--linux-s390_64.zip -EOF - exit 1 -fi - -TARGET=$1 -VERSION_NUMBER=$2 - -# pairs. -declare -a FILE_NAMES=( \ - win32.zip windows-x86_32.exe \ - win64.zip windows-x86_64.exe \ - osx-aarch_64.zip osx-aarch_64.exe \ - osx-x86_64.zip osx-x86_64.exe \ - linux-x86_32.zip linux-x86_32.exe \ - linux-x86_64.zip linux-x86_64.exe \ - linux-aarch_64.zip linux-aarch_64.exe \ - linux-ppcle_64.zip linux-ppcle_64.exe \ - linux-s390_64.zip linux-s390_64.exe \ -) - -# List of all well-known types to be included. -declare -a WELL_KNOWN_TYPES=( \ - google/protobuf/descriptor.proto \ - google/protobuf/any.proto \ - google/protobuf/api.proto \ - google/protobuf/duration.proto \ - google/protobuf/empty.proto \ - google/protobuf/field_mask.proto \ - google/protobuf/source_context.proto \ - google/protobuf/struct.proto \ - google/protobuf/timestamp.proto \ - google/protobuf/type.proto \ - google/protobuf/wrappers.proto \ - google/protobuf/compiler/plugin.proto \ -) - -set -e - -# A temporary working directory to put all files. -DIR=$(mktemp -d) - -# Copy over well-known types. -mkdir -p ${DIR}/include/google/protobuf/compiler -for PROTO in ${WELL_KNOWN_TYPES[@]}; do - cp -f ../src/${PROTO} ${DIR}/include/${PROTO} -done - -# Create a readme file. -cat < ${DIR}/readme.txt -Protocol Buffers - Google's data interchange format -Copyright 2008 Google Inc. -https://developers.google.com/protocol-buffers/ - -This package contains a precompiled binary version of the protocol buffer -compiler (protoc). This binary is intended for users who want to use Protocol -Buffers in languages other than C++ but do not want to compile protoc -themselves. To install, simply place this binary somewhere in your PATH. - -If you intend to use the included well known types then don't forget to -copy the contents of the 'include' directory somewhere as well, for example -into '/usr/local/include/'. - -Please refer to our official github site for more installation instructions: - https://github.com/protocolbuffers/protobuf -EOF - -mkdir -p dist -mkdir -p ${DIR}/bin -# Create a zip file for each binary. -for((i=0;i<${#FILE_NAMES[@]};i+=2));do - ZIP_NAME=${FILE_NAMES[$i]} - if [ ${ZIP_NAME:0:3} = "win" ]; then - BINARY="$TARGET.exe" - else - BINARY="$TARGET" - fi - BINARY_NAME=${FILE_NAMES[$(($i+1))]} - BINARY_URL=https://repo1.maven.org/maven2/com/google/protobuf/$TARGET/${VERSION_NUMBER}/$TARGET-${VERSION_NUMBER}-${BINARY_NAME} - if ! wget ${BINARY_URL} -O ${DIR}/bin/$BINARY &> /dev/null; then - echo "[ERROR] Failed to download ${BINARY_URL}" >&2 - echo "[ERROR] Skipped $TARGET-${VERSION_NAME}-${ZIP_NAME}" >&2 - continue - fi - TARGET_ZIP_FILE=`pwd`/dist/$TARGET-${VERSION_NUMBER}-${ZIP_NAME} - pushd $DIR &> /dev/null - chmod +x bin/$BINARY - if [ "$TARGET" = "protoc" ]; then - zip -r ${TARGET_ZIP_FILE} include bin readme.txt &> /dev/null - else - zip -r ${TARGET_ZIP_FILE} bin &> /dev/null - fi - rm bin/$BINARY - popd &> /dev/null - echo "[INFO] Successfully created ${TARGET_ZIP_FILE}" -done diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml deleted file mode 100644 index 19e700d4d3..0000000000 --- a/protoc-artifacts/pom.xml +++ /dev/null @@ -1,144 +0,0 @@ - - - 4.0.0 - - com.google - google - 5 - - com.google.protobuf - protoc - 3.21.6 - pom - Protobuf Compiler - - Protobuf Compiler (protoc) is a compiler for .proto files. It generates - language-specific code for Protobuf messages and RPC interfaces. - - 2008 - https://developers.google.com/protocol-buffers/ - - - BSD-3-Clause - https://opensource.org/licenses/BSD-3-Clause - repo - - - - https://github.com/protocolbuffers/protobuf - - scm:git:https://github.com/protocolbuffers/protobuf.git - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 1.8 - - - attach-artifacts - package - - attach-artifact - - - - - ${basedir}/target/linux/x86_64/protoc.exe - linux-x86_64 - exe - - - ${basedir}/target/linux/x86_32/protoc.exe - linux-x86_32 - exe - - - ${basedir}/target/windows/x86_64/protoc.exe - windows-x86_64 - exe - - - ${basedir}/target/windows/x86_32/protoc.exe - windows-x86_32 - exe - - - ${basedir}/target/osx/x86_64/protoc.exe - osx-x86_64 - exe - - - ${basedir}/target/osx/aarch_64/protoc.exe - osx-aarch_64 - exe - - - ${basedir}/target/linux/aarch_64/protoc.exe - linux-aarch_64 - exe - - - ${basedir}/target/linux/ppcle_64/protoc.exe - linux-ppcle_64 - exe - - - ${basedir}/target/linux/s390_64/protoc.exe - linux-s390_64 - exe - - - - - - - - - - - release - - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.3 - true - - sonatype-nexus-staging - https://oss.sonatype.org/ - true - false - ${staging.repository} - - - - - - - diff --git a/protoc-artifacts/scl-enable-devtoolset.sh b/protoc-artifacts/scl-enable-devtoolset.sh deleted file mode 100755 index 30895585f3..0000000000 --- a/protoc-artifacts/scl-enable-devtoolset.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -eu -o pipefail - -quote() { - local arg - for arg in "$@"; do - printf "'" - printf "%s" "$arg" | sed -e "s/'/'\\\\''/g" - printf "' " - done -} - -exec scl enable devtoolset-2 "$(quote "$@")"