From ade435e981ad59d600231811979f082094f95b96 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 27 Feb 2018 17:40:14 +0100 Subject: [PATCH] add build_packages and distribtest kokoro jobs --- .../internal_ci/linux/grpc_build_packages.cfg | 26 ++++++++++++++ .../internal_ci/linux/grpc_build_packages.sh | 35 +++++++++++++++++++ tools/internal_ci/linux/grpc_distribtests.cfg | 26 ++++++++++++++ tools/internal_ci/linux/grpc_distribtests.sh | 35 +++++++++++++++++++ tools/internal_ci/macos/grpc_distribtests.cfg | 27 ++++++++++++++ tools/internal_ci/macos/grpc_distribtests.sh | 27 ++++++++++++++ .../windows/grpc_build_packages.bat | 35 +++++++++++++++++++ .../windows/grpc_build_packages.cfg | 26 ++++++++++++++ .../internal_ci/windows/grpc_distribtests.bat | 35 +++++++++++++++++++ .../internal_ci/windows/grpc_distribtests.cfg | 26 ++++++++++++++ 10 files changed, 298 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_build_packages.cfg create mode 100644 tools/internal_ci/linux/grpc_build_packages.sh create mode 100644 tools/internal_ci/linux/grpc_distribtests.cfg create mode 100644 tools/internal_ci/linux/grpc_distribtests.sh create mode 100644 tools/internal_ci/macos/grpc_distribtests.cfg create mode 100644 tools/internal_ci/macos/grpc_distribtests.sh create mode 100644 tools/internal_ci/windows/grpc_build_packages.bat create mode 100644 tools/internal_ci/windows/grpc_build_packages.cfg create mode 100644 tools/internal_ci/windows/grpc_distribtests.bat create mode 100644 tools/internal_ci/windows/grpc_distribtests.cfg diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg new file mode 100644 index 00000000000..6a4a163dfca --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_build_packages.sh b/tools/internal_ci/linux/grpc_build_packages.sh new file mode 100644 index 00000000000..9f65d8ed5f4 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_packages.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +set -ex + +# Move artifacts generated by the previous step in the build chain to a place +# where they can be accessed from within a docker container that builds +# the packages +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +ls -R input_artifacts || true + +tools/run_tests/task_runner.py -f package linux -j 6 diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg new file mode 100644 index 00000000000..0f1d79355a6 --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_distribtests.sh b/tools/internal_ci/linux/grpc_distribtests.sh new file mode 100644 index 00000000000..47e4bf877a2 --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +set -ex + +# Move packages generated by the previous step in the build chain to a place +# where they can be accessed from within a docker container that run the +# distribtests +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +ls -R input_artifacts || true + +tools/run_tests/task_runner.py -f distribtest linux -j 6 diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg new file mode 100644 index 00000000000..ae88f39b903 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests.cfg @@ -0,0 +1,27 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/grpc_distribtests.sh b/tools/internal_ci/macos/grpc_distribtests.sh new file mode 100644 index 00000000000..59ea833eeca --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +# Move packages generated by the previous step in the build chain. +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +ls -R input_artifacts || true + +tools/run_tests/task_runner.py -f distribtest macos diff --git a/tools/internal_ci/windows/grpc_build_packages.bat b/tools/internal_ci/windows/grpc_build_packages.bat new file mode 100644 index 00000000000..23ba365e408 --- /dev/null +++ b/tools/internal_ci/windows/grpc_build_packages.bat @@ -0,0 +1,35 @@ +@rem Copyright 2017 gRPC authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder +@rem TODO(jtattermusch): get rid of this hack +rename C:\Python27_32bit Python27_32bits +rename C:\Python34_32bit Python34_32bits +rename C:\Python35_32bit Python35_32bits +rename C:\Python36_32bit Python36_32bits + +@rem enter repo root +cd /d %~dp0\..\..\.. + +call tools/internal_ci/helper_scripts/prepare_build_windows.bat + +@rem Move artifacts generated by the previous step in the build chain. +powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" +dir input_artifacts + +python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg new file mode 100644 index 00000000000..65a8b1eef33 --- /dev/null +++ b/tools/internal_ci/windows/grpc_build_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/windows/grpc_distribtests.bat b/tools/internal_ci/windows/grpc_distribtests.bat new file mode 100644 index 00000000000..d6d50b684f5 --- /dev/null +++ b/tools/internal_ci/windows/grpc_distribtests.bat @@ -0,0 +1,35 @@ +@rem Copyright 2017 gRPC authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder +@rem TODO(jtattermusch): get rid of this hack +rename C:\Python27_32bit Python27_32bits +rename C:\Python34_32bit Python34_32bits +rename C:\Python35_32bit Python35_32bits +rename C:\Python36_32bit Python36_32bits + +@rem enter repo root +cd /d %~dp0\..\..\.. + +call tools/internal_ci/helper_scripts/prepare_build_windows.bat + +@rem Move packages generated by the previous step in the build chain. +powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts" +dir input_artifacts + +python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg new file mode 100644 index 00000000000..1766e601e51 --- /dev/null +++ b/tools/internal_ci/windows/grpc_distribtests.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +}