Migrate remaining cross-architecture tests to GHA

PiperOrigin-RevId: 506995031

.github/actions/cross-compile-protoc/action.yml

@@ -0,0 +1,38 @@
+name: 'Cross-compile protoc'
+description: 'Produces a cross-compiled protoc binary for a target architecture'
+inputs:
+  credentials:
+    required: true
+    description: The GCP credentials to use for reading the docker image
+    type: string
+  architecture:
+    required: true
+    description: The target architecture to build for
+    type: string
+outputs:
+  protoc:
+    description: "Cross-compiled protoc location.  Also output to $PROTOC"
+    value: ${{ steps.output.outputs.protoc }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Cross compile protoc for ${{ inputs.architecture }}
+      uses: ./.github/actions/bazel-docker
+      with:
+        credentials: ${{ inputs.credentials }}
+        bazel-cache: xcompile-protoc/${{ inputs.architecture }}
+        bash: |
+          bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS
+          cp bazel-bin/protoc .
+
+    - name: Set protoc environment variable
+      shell: bash
+      run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV
+
+    - name: Extract binary
+      id: output
+      shell: bash
+      run: |
+        mv protoc $PROTOC
+        echo "protoc=$PROTOC" >> $GITHUB_OUTPUT

.github/actions/docker/action.yml

@@ -13,6 +13,14 @@ inputs:
     default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
     description: "The docker image to use"
     type: string
+  platform:
+    required: false
+    description: "The platform to use for the image"
+    type: string
+  skip-staleness-check:
+    required: false
+    description: "Skip staleness checks"
+    type: boolean
 
 runs:
   using: 'composite'
@@ -21,6 +29,7 @@ runs:
       uses: ./.github/actions/internal/setup-runner
 
     - name: Update stale files using Bazel
+      if: ${{ !inputs.skip-staleness-check }}
       uses: ./.github/actions/bazel-docker
       with:
         image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
@@ -28,8 +37,14 @@ runs:
         bazel-cache: regenerate-stale-files
         bash: ./regenerate_stale_files.sh $BAZEL_FLAGS
 
+    - name: Generate docker flags
+      if: inputs.platform
+      shell: bash
+      run: echo "DOCKER_RUN_FLAGS=--platform ${{inputs.platform}}" >> $GITHUB_ENV
+
     - name: Run Docker
       uses: ./.github/actions/internal/docker-run
       with:
         image: ${{ inputs.image }}
         command: ${{ inputs.command }}
+        run-flags: ${{ env.DOCKER_RUN_FLAGS }}

.github/workflows/test_cpp.yml

@@ -80,6 +80,16 @@ jobs:
               -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
               -Dprotobuf_BUILD_CONFORMANCE=ON
               -DCMAKE_CXX_STANDARD=14
+          - name: 32-bit
+            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:6651a299483f7368876db7aed0802ad4ebf038d626d8995ba7df08978ff43210
+            platform: linux/386
+            command: >-
+              /bin/bash -c '
+              cd /workspace;
+              cmake . -DCMAKE_CXX_STANDARD=14;
+              cmake --build . --parallel 20;
+              ctest --verbose --parallel 20'
+
     name: Linux CMake ${{ matrix.name}}
     runs-on: ubuntu-latest
     steps:
@@ -91,7 +101,8 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/docker
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335
+          image: ${{ matrix.image || 'us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335' }}
+          platform: ${{ matrix.platform }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: ${{ matrix.command }}
   non-linux:

.github/workflows/test_csharp.yml

@@ -44,3 +44,43 @@ jobs:
 
       - name: Run Tests
         run: dotnet test csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj
+
+  linux-aarch64:
+    name: Linux aarch64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Build protobuf C# tests under x86_64 docker image
+        # Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
+        # (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
+        uses: ./.github/actions/docker
+        with:
+          image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          command: >-
+            /bin/bash -c '
+            DOTNET_CLI_TELEMETRY_OPTOUT=true
+            DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true
+            dotnet publish -c Release -f net60 /workspace/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj'
+
+      - name: Use an actual aarch64 docker image to run protobuf C# tests with an emulator
+        # "dotnet vstest" allows running tests from a pre-built project.
+        # * mount the protobuf root as /work to be able to access the crosscompiled files
+        # * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+        #   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+        #   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+        #   we just run map the user's home to a throwaway temporary directory
+        uses: ./.github/actions/docker
+        with:
+          image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8
+          skip-staleness-check: true
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          command: >-
+            /bin/bash -c '
+            DOTNET_CLI_TELEMETRY_OPTOUT=true
+            DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true
+            dotnet vstest /workspace/csharp/src/Google.Protobuf.Test/bin/Release/net60/publish/Google.Protobuf.Test.dll'

.github/workflows/test_php.yml

@@ -49,6 +49,87 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: ${{ matrix.command }}
 
+  linux-32bit:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        version: ['7.4', '8.0']
+        suffix: [ '', '-zts']
+        test: ['test', 'test_c']
+        exclude:
+          - suffix: '-zts'
+            test: 'test'
+        include:
+          - suffix: '-zts'
+            suffix_name: ' Thread Safe'
+          - test: 'test_c'
+            test_name: ' Extension'
+
+    name: Linux 32-bit ${{ matrix.version}}${{ matrix.suffix_name }}${{ matrix.test_name }}
+    runs-on: ubuntu-latest
+    env:
+      image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:6651a299483f7368876db7aed0802ad4ebf038d626d8995ba7df08978ff43210
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          submodules: recursive
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Cross compile protoc for i386
+        id: cross-compile
+        uses: ./.github/actions/cross-compile-protoc
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          architecture: linux-i386
+
+      - name: Run tests
+        uses: ./.github/actions/docker
+        with:
+          image: ${{ env.image }}
+          skip-staleness-check: true
+          platform: linux/386
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          command: >-
+            /bin/bash -c '
+            cd /workspace/php && php -v && php -m;
+            composer update --ignore-platform-reqs;
+            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
+            PATH="/usr/local/php-${{ matrix.version }}${{matrix.suffix}}/bin:$PATH"
+            composer ${{ matrix.test }}'
+
+  linux-aarch64:
+    name: Linux aarch64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Cross compile protoc for aarch64
+        id: cross-compile
+        uses: ./.github/actions/cross-compile-protoc
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          architecture: linux-aarch64
+
+      - name: Run tests
+        uses: ./.github/actions/docker
+        with:
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310
+          platform: linux/arm64
+          skip-staleness-check: true
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          command: >-
+            -c '
+            cd php;
+            composer update --ignore-platform-reqs;
+            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
+            composer test;
+            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
+            composer test_c'
+
   macos:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.

.github/workflows/test_ruby.yml

@@ -34,11 +34,42 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/bazel-docker
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
+          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
           bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
 
+  linux-aarch64:
+    name: Linux aarch64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Cross compile protoc for aarch64
+        id: cross-compile
+        uses: ./.github/actions/cross-compile-protoc
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          architecture: linux-aarch64
+
+      - name: Run tests
+        uses: ./.github/actions/docker
+        with:
+          image: arm64v8/ruby:2.7.3-buster
+          skip-staleness-check: true
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          command: >-
+            /bin/bash -c '
+            gem install bundler;
+            cd /workspace/ruby;
+            bundle;
+            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake;
+            rake clobber_package gem;
+            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake test'
+
   macos:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.

kokoro/linux/aarch64/test_php_aarch64.sh

@@ -5,6 +5,8 @@ set -ex
 # go to the repo root
 cd $(dirname $0)/../../..
 
+git submodule update --init --recursive
+
 # there is no php testing docker image readily available, so we build
 # our own. It's a aarch64 image, but that's fine since qemu will
 # automatically be used to run the commands in the dockerfile.

php/tests/compile_extension.sh

@@ -4,9 +4,6 @@ set -ex
 
 cd $(dirname $0)/..
 
-# Pull in dependencies.
-git submodule update --init --recursive
-
 # utf8_range has to live in the base third_party directory.
 # We copy it into the ext/google/protobuf directory for the build
 # (and for the release to PECL).