diff --git a/.github/actions/cross-compile-protoc/action.yml b/.github/actions/cross-compile-protoc/action.yml new file mode 100644 index 0000000000..23660af705 --- /dev/null +++ b/.github/actions/cross-compile-protoc/action.yml @@ -0,0 +1,38 @@ +name: 'Cross-compile protoc' +description: 'Produces a cross-compiled protoc binary for a target architecture' +inputs: + credentials: + required: true + description: The GCP credentials to use for reading the docker image + type: string + architecture: + required: true + description: The target architecture to build for + type: string +outputs: + protoc: + description: "Cross-compiled protoc location. Also output to $PROTOC" + value: ${{ steps.output.outputs.protoc }} + +runs: + using: 'composite' + steps: + - name: Cross compile protoc for ${{ inputs.architecture }} + uses: ./.github/actions/bazel-docker + with: + credentials: ${{ inputs.credentials }} + bazel-cache: xcompile-protoc/${{ inputs.architecture }} + bash: | + bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS + cp bazel-bin/protoc . + + - name: Set protoc environment variable + shell: bash + run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV + + - name: Extract binary + id: output + shell: bash + run: | + mv protoc $PROTOC + echo "protoc=$PROTOC" >> $GITHUB_OUTPUT diff --git a/.github/actions/docker/action.yml b/.github/actions/docker/action.yml index 93bab6be70..54de3f908e 100644 --- a/.github/actions/docker/action.yml +++ b/.github/actions/docker/action.yml @@ -13,6 +13,14 @@ inputs: default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 description: "The docker image to use" type: string + platform: + required: false + description: "The platform to use for the image" + type: string + skip-staleness-check: + required: false + description: "Skip staleness checks" + type: boolean runs: using: 'composite' @@ -21,6 +29,7 @@ runs: uses: ./.github/actions/internal/setup-runner - name: Update stale files using Bazel + if: ${{ !inputs.skip-staleness-check }} uses: ./.github/actions/bazel-docker with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 @@ -28,8 +37,14 @@ runs: bazel-cache: regenerate-stale-files bash: ./regenerate_stale_files.sh $BAZEL_FLAGS + - name: Generate docker flags + if: inputs.platform + shell: bash + run: echo "DOCKER_RUN_FLAGS=--platform ${{inputs.platform}}" >> $GITHUB_ENV + - name: Run Docker uses: ./.github/actions/internal/docker-run with: image: ${{ inputs.image }} command: ${{ inputs.command }} + run-flags: ${{ env.DOCKER_RUN_FLAGS }} diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index ce33686202..66c6677424 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -80,6 +80,16 @@ jobs: -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 + - name: 32-bit + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:6651a299483f7368876db7aed0802ad4ebf038d626d8995ba7df08978ff43210 + platform: linux/386 + command: >- + /bin/bash -c ' + cd /workspace; + cmake . -DCMAKE_CXX_STANDARD=14; + cmake --build . --parallel 20; + ctest --verbose --parallel 20' + name: Linux CMake ${{ matrix.name}} runs-on: ubuntu-latest steps: @@ -91,7 +101,8 @@ jobs: - name: Run tests uses: ./.github/actions/docker with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335 + image: ${{ matrix.image || 'us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335' }} + platform: ${{ matrix.platform }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} non-linux: diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index 4b7fad8ea6..022d4f1f6c 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -44,3 +44,43 @@ jobs: - name: Run Tests run: dotnet test csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj + + linux-aarch64: + name: Linux aarch64 + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + + - name: Build protobuf C# tests under x86_64 docker image + # Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory + # (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run) + uses: ./.github/actions/docker + with: + image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + /bin/bash -c ' + DOTNET_CLI_TELEMETRY_OPTOUT=true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true + dotnet publish -c Release -f net60 /workspace/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj' + + - name: Use an actual aarch64 docker image to run protobuf C# tests with an emulator + # "dotnet vstest" allows running tests from a pre-built project. + # * mount the protobuf root as /work to be able to access the crosscompiled files + # * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force + # running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user + # otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, + # we just run map the user's home to a throwaway temporary directory + uses: ./.github/actions/docker + with: + image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 + skip-staleness-check: true + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + /bin/bash -c ' + DOTNET_CLI_TELEMETRY_OPTOUT=true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true + dotnet vstest /workspace/csharp/src/Google.Protobuf.Test/bin/Release/net60/publish/Google.Protobuf.Test.dll' diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index b492b046f8..dfcf02537b 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -49,6 +49,87 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} + linux-32bit: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + version: ['7.4', '8.0'] + suffix: [ '', '-zts'] + test: ['test', 'test_c'] + exclude: + - suffix: '-zts' + test: 'test' + include: + - suffix: '-zts' + suffix_name: ' Thread Safe' + - test: 'test_c' + test_name: ' Extension' + + name: Linux 32-bit ${{ matrix.version}}${{ matrix.suffix_name }}${{ matrix.test_name }} + runs-on: ubuntu-latest + env: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:6651a299483f7368876db7aed0802ad4ebf038d626d8995ba7df08978ff43210 + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + + - name: Cross compile protoc for i386 + id: cross-compile + uses: ./.github/actions/cross-compile-protoc + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + architecture: linux-i386 + + - name: Run tests + uses: ./.github/actions/docker + with: + image: ${{ env.image }} + skip-staleness-check: true + platform: linux/386 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + /bin/bash -c ' + cd /workspace/php && php -v && php -m; + composer update --ignore-platform-reqs; + PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} + PATH="/usr/local/php-${{ matrix.version }}${{matrix.suffix}}/bin:$PATH" + composer ${{ matrix.test }}' + + linux-aarch64: + name: Linux aarch64 + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + + - name: Cross compile protoc for aarch64 + id: cross-compile + uses: ./.github/actions/cross-compile-protoc + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + architecture: linux-aarch64 + + - name: Run tests + uses: ./.github/actions/docker + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310 + platform: linux/arm64 + skip-staleness-check: true + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + -c ' + cd php; + composer update --ignore-platform-reqs; + PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} + composer test; + PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} + composer test_c' + macos: strategy: fail-fast: false # Don't cancel all jobs if one fails. diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 54194e6f2c..e22bf8a356 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -34,11 +34,42 @@ jobs: - name: Run tests uses: ./.github/actions/bazel-docker with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION + linux-aarch64: + name: Linux aarch64 + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + + - name: Cross compile protoc for aarch64 + id: cross-compile + uses: ./.github/actions/cross-compile-protoc + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + architecture: linux-aarch64 + + - name: Run tests + uses: ./.github/actions/docker + with: + image: arm64v8/ruby:2.7.3-buster + skip-staleness-check: true + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + /bin/bash -c ' + gem install bundler; + cd /workspace/ruby; + bundle; + PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake; + rake clobber_package gem; + PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake test' + macos: strategy: fail-fast: false # Don't cancel all jobs if one fails. diff --git a/kokoro/linux/aarch64/test_php_aarch64.sh b/kokoro/linux/aarch64/test_php_aarch64.sh index b11ef43071..3eb8fa45e2 100755 --- a/kokoro/linux/aarch64/test_php_aarch64.sh +++ b/kokoro/linux/aarch64/test_php_aarch64.sh @@ -5,6 +5,8 @@ set -ex # go to the repo root cd $(dirname $0)/../../.. +git submodule update --init --recursive + # there is no php testing docker image readily available, so we build # our own. It's a aarch64 image, but that's fine since qemu will # automatically be used to run the commands in the dockerfile. diff --git a/php/tests/compile_extension.sh b/php/tests/compile_extension.sh index a645ed60da..5f7a6cce69 100755 --- a/php/tests/compile_extension.sh +++ b/php/tests/compile_extension.sh @@ -4,9 +4,6 @@ set -ex cd $(dirname $0)/.. -# Pull in dependencies. -git submodule update --init --recursive - # utf8_range has to live in the base third_party directory. # We copy it into the ext/google/protobuf directory for the build # (and for the release to PECL).