Add Python GCF Distribtest (#29303)

* Initial GCF distribtest

* Tenatively hook up to CI

* Try again

* Allow dev0 artifacts

* Fix invocation path

* Update gcloud

* Add a 3.8 artifact for presubmits

* And 3.9 too

* Put test files back to normal

* Formatting/linting

* Copyright

* That copyright script doesn't work with shebangs

* Review comments

* Try to create latest-manylinux label

* Accidentally a letter

* Add Python 3.7 manylinux 2014 to presubmit

* Revert CI config file used for test

* Review comments

* Yapf

* Re-add presubmit wheel

* Review comments

test/distrib/gcf/python/.gcloudignore

@@ -0,0 +1,19 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+*.sh
+*.md
+*.whl
+requirements.txt.base

test/distrib/gcf/python/.gitignore

@@ -0,0 +1 @@
+requirements.txt

test/distrib/gcf/python/README.md

@@ -0,0 +1,12 @@
+# Python Google Cloud Functions Distribtest
+
+This distribtest acts as a smoke test for usage of the `grpcio` Python wheel in
+the GCF environment. This test is dependent on two long-lived GCP resources:
+
+- `gcf-distribtest-topic` Pub Sub Topic with default configuration.
+- `grpc-gcf-distribtest` GCS Bucket. This bucket has 1 day TTL on all artifacts.
+
+
+All Functions _should_ be deleted by the test under normal circumstances. In
+case anything goes wrong with this process, a `cleanup.sh` script is provided to
+delete any dangling test functions.

test/distrib/gcf/python/cleanup.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd "$(dirname "$0")"
+
+# Shellcheck cant find the included file.
+# shellcheck disable=SC1091
+source common.sh
+
+set -euxo pipefail
+
+gcloud functions list | grep "${FUNCTION_PREFIX}" | awk '{print $1;}' |  xargs -n1 gcloud functions delete

test/distrib/gcf/python/common.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Shellcheck can't track usage across files.
+# shellcheck disable=SC2034
+FUNCTION_PREFIX="grpc-gcf-distribtest"

test/distrib/gcf/python/main.py

@@ -0,0 +1,32 @@
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functions_framework
+from google.cloud import pubsub_v1
+
+ps_client = pubsub_v1.PublisherClient()
+_PROJECT_ID = "grpc-testing"
+_PUBSUB_TOPIC = 'gcf-distribtest-topic'
+
+
+@functions_framework.http
+def test_publish(request):
+    topic_path = ps_client.topic_path(_PROJECT_ID, _PUBSUB_TOPIC)
+    message = '{"function": "TEST"}'
+    message_bytes = message.encode('utf-8')
+
+    for _ in range(100):
+        future = ps_client.publish(topic_path, data=message_bytes)
+
+    return "ok", 200

test/distrib/gcf/python/requirements.txt.base

@@ -0,0 +1 @@
+google-cloud-pubsub>=0.0.dev0

test/distrib/gcf/python/run.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euxo pipefail
+
+cd "$(dirname "$0")"
+
+ARTIFACT_DIRECTORY="$1"
+
+BUCKET_NAME="grpc-gcf-distribtests"
+
+RUN_ID=$(uuidgen)
+
+FAILED_RUNTIMES=""
+
+# This is the only programmatic way to get access to the list of runtimes.
+# While hacky, it's better than the alternative -- manually upgrading a
+# hand-curated list every few months.
+RUNTIMES=$(gcloud functions deploy --help | grep -Eo "python[0-9]+" | sort | uniq)
+
+while read -r RUNTIME; do
+  BARE_VERSION=${RUNTIME//python/}
+
+  # We sort to get the latest manylinux version.
+  ARTIFACT=$(find "${ARTIFACT_DIRECTORY}" -regex '.*grpcio-[0-9\.]+.+-cp'"${BARE_VERSION}"'-cp'"${BARE_VERSION}"'m?-manylinux.+x86_64\.whl' | sort -r | head -n 1)
+  ARTIFACT_BASENAME=$(basename "${ARTIFACT}")
+
+  # Upload artifact to GCS so GCF can access it.
+  # A 1 day retention policy is active on this bucket.
+  gsutil cp "${ARTIFACT}" "gs://${BUCKET_NAME}/${RUN_ID}/${ARTIFACT_BASENAME}"
+
+  echo "Testing runtime ${RUNTIME} with artifact ${ARTIFACT_BASENAME}"
+  ./run_single.sh "${RUNTIME}" "https://storage.googleapis.com/${BUCKET_NAME}/${RUN_ID}/${ARTIFACT_BASENAME}" || FAILED_RUNTIMES="${FAILED_RUNTIMES} ${RUNTIME}"
+done<<<"${RUNTIMES}"
+
+if [ "$FAILED_RUNTIMES" != "" ]
+then
+  echo "GCF Distribtest failed: Failing runtimes: ${FAILED_RUNTIMES}"
+  exit 1
+fi

test/distrib/gcf/python/run_single.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd "$(dirname "$0")"
+
+# Shellcheck cant find the included file.
+# shellcheck disable=SC1091
+source common.sh
+
+set -euxo pipefail
+
+RUNTIME="$1"
+ARTIFACT_URL="$2"
+
+REQUEST_COUNT=20
+LOG_QUIESCE_SECONDS=10
+
+rm -f requirements.txt
+cp requirements.txt.base requirements.txt
+echo "${ARTIFACT_URL}" >>requirements.txt
+
+# Generate Function name.
+FUNCTION_NAME="${FUNCTION_PREFIX}-$(uuidgen)"
+
+function cleanup() {
+  # Wait for logs to quiesce.
+  sleep "${LOG_QUIESCE_SECONDS}"
+  gcloud functions logs read "${FUNCTION_NAME}" || true
+  (yes || true) | gcloud functions delete "${FUNCTION_NAME}"
+}
+
+trap cleanup SIGINT SIGTERM EXIT
+
+# Deploy
+DEPLOY_OUTPUT=$(gcloud functions deploy "${FUNCTION_NAME}" --entry-point test_publish --runtime "${RUNTIME}" --trigger-http --allow-unauthenticated)
+HTTP_URL=$(echo "${DEPLOY_OUTPUT}" | grep "url: " | awk '{print $2;}')
+
+# Send Requests
+for _ in $(seq 1 "${REQUEST_COUNT}"); do
+  GCP_IDENTITY_TOKEN=$(gcloud auth print-identity-token 2>/dev/null);
+  curl -v -H "Authorization: Bearer $GCP_IDENTITY_TOKEN" "${HTTP_URL}"
+done

tools/internal_ci/linux/grpc_distribtests_gcp_python.cfg

@@ -0,0 +1,26 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_gcp_python.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/linux/grpc_distribtests_gcp_python.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+# some distribtests use a pre-registered binfmt_misc hook
+# to automatically execute foreign binaries (such as aarch64)
+# under qemu emulator.
+source tools/internal_ci/helper_scripts/prepare_qemu_rc
+
+# configure ccache
+source tools/internal_ci/helper_scripts/prepare_ccache_rc
+
+# Build all python linux artifacts (this step actually builds all the binary wheels and source archives)
+tools/run_tests/task_runner.py -f artifact linux python latest-manylinux ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+rm -rf artifacts_from_build_artifacts_step
+mv artifacts artifacts_from_build_artifacts_step || true
+
+# This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/
+tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+# in addition to that, preserve the contents of "artifacts" directory since we want kokoro
+# to upload its contents as job output artifacts
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+
+INPUT_ARTIFACTS=$(realpath input_artifacts)
+
+(yes || true) | gcloud components update
+
+# TODO(rbellevi): Run in a docker image.
+test/distrib/gcf/python/run.sh "$INPUT_ARTIFACTS"
+
+tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/run_tests/artifacts/artifact_targets.py

@@ -22,6 +22,8 @@ import sys
 sys.path.insert(0, os.path.abspath('..'))
 import python_utils.jobset as jobset
 
+_LATEST_MANYLINUX = "manylinux2014"
+
 
 def create_docker_jobspec(name,
                           dockerfile_dir,
@@ -107,6 +109,8 @@ class PythonArtifact:
         if presubmit:
             self.labels.append('presubmit')
         self.py_version = py_version
+        if platform == _LATEST_MANYLINUX:
+            self.labels.append('latest-manylinux')
         if 'manylinux' in platform:
             self.labels.append('linux')
         if 'linux_extra' in platform: