From 3dbaf171f9d9480605223b9bf16baf1a2b946c20 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Mon, 16 Mar 2015 18:35:53 -0700 Subject: [PATCH 1/4] Adds a dockerfile and script for building a tar archive containing protoc and the grpc plugins --- tools/dockerfile/grpc_dist_proto/Dockerfile | 73 +++++++++++++++++++++ tools/gce_setup/grpc_docker.sh | 52 +++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 tools/dockerfile/grpc_dist_proto/Dockerfile diff --git a/tools/dockerfile/grpc_dist_proto/Dockerfile b/tools/dockerfile/grpc_dist_proto/Dockerfile new file mode 100644 index 00000000000..a15b7ca4a61 --- /dev/null +++ b/tools/dockerfile/grpc_dist_proto/Dockerfile @@ -0,0 +1,73 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Dockerfile to build protoc and plugins for inclusion in a release. +FROM grpc/base + +# Install tools needed for building protoc. +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev + +# Get the protobuf source from GitHub. +RUN mkdir -p /var/local/git +RUN git clone https://github.com/google/protobuf.git /var/local/git/protobuf + +# Build the protobuf library statically and install to /tmp/protoc_static. +WORKDIR /var/local/git/protobuf +RUN ./autogen.sh && \ + ./configure --disable-shared --prefix=/tmp/protoc_static \ + LDFLAGS="-lgcc_eh -static-libgcc -static-libstdc++" && \ + make -j12 && make check && make install + +# Build the protobuf library dynamically and install to /usr/local. +WORKDIR /var/local/git/protobuf +RUN ./autogen.sh && \ + ./configure --prefix=/usr/local && \ + make -j12 && make check && make install + +# Build the grpc plugins. +RUN git clone https://github.com/google/grpc.git /var/local/git/grpc +WORKDIR /var/local/git/grpc +RUN LDFLAGS=-static make plugins + +# Create an archive containing all the generated binaries. +RUN mkdir /tmp/proto_bins_root +RUN cp -v bins/opt/* /tmp/proto_bins_root +RUN cp -v /tmp/protoc_static/bin/protoc /tmp/proto_bins_root +RUN cd /tmp/proto_bins_root && \ + tar -czf /tmp/proto-bins-linux-$(uname -m).tar.gz * + +# List the tar contents: provides a way to visually confirm that the contents +# are correct. +RUN echo 'proto-bins-linux-tar-$(uname -m) contents:' && \ + tar -ztf /tmp/proto-bins-linux-$(uname -m).tar.gz + + + + + diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 619eff56d62..44b5b459de9 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -663,6 +663,58 @@ grpc_show_servers() { gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" } +_grpc_build_proto_bins_args() { + [[ -n $1 ]] && { # host + host=$1 + shift + } || { + host='grpc-docker-builder' + } +} + +# grpc_build_proto_bins +# +# - rebuilds the dist_proto docker image +# * doing this builds the protoc and the ruby, python and cpp bins statically +# +# - runs a docker command that copies the built protos to the GCE host +# - copies the built protos to the local machine +grpc_build_proto_bins() { + _grpc_ensure_gcloud_ssh || return 1; + + # declare vars local so that they don't pollute the shell environment + # where they this func is used. + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone + # set by _grpc_build_proto_bins_args + local host + + # set the project zone and check that all necessary args are provided + _grpc_set_project_and_zone -f _grpc_build_proto_bins_args "$@" || return 1 + gce_has_instance $grpc_project $host || return 1; + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + + # rebuild the dist_proto image + local label='dist_proto' + grpc_update_image -- -h $host $label || return 1 + + # run a command to copy the generated output to the local machine + local docker_prefix='sudo docker run -v /tmp:/tmp/proto_bins_out' + local tar_name='proto-bins-linux-x86_64.tar.gz' + local cp_cmd="cp /tmp/$tar_name /tmp/proto_bins_out" + local cmd="$docker_prefix grpc/$label $cp_cmd" + local ssh_cmd="bash -l -c \"$cmd\"" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" || return 1 + + # copy the tar.gz locally + local rmt_tar="$host:/tmp/$tar_name" + local local_copy="$(pwd)/$tar_name" + gcloud compute copy-files $rmt_tar $local_copy $project_opt $zone_opt || return 1 +} + _grpc_launch_servers_args() { [[ -n $1 ]] && { # host host=$1 From fb23f1c0e4a38354e28fe3776126eb8655561064 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 18 Mar 2015 11:42:53 -0700 Subject: [PATCH 2/4] Adds a file containing the version --- tools/dockerfile/grpc_dist_proto/version.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 tools/dockerfile/grpc_dist_proto/version.txt diff --git a/tools/dockerfile/grpc_dist_proto/version.txt b/tools/dockerfile/grpc_dist_proto/version.txt new file mode 100644 index 00000000000..8f0916f768f --- /dev/null +++ b/tools/dockerfile/grpc_dist_proto/version.txt @@ -0,0 +1 @@ +0.5.0 From 64af6830f38b120139deba09215b2dc8786357a3 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 18 Mar 2015 11:43:39 -0700 Subject: [PATCH 3/4] Corrects some spelling mistakes --- tools/gce_setup/grpc_docker.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 44b5b459de9..7ffd5b2203f 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -560,7 +560,7 @@ grpc_sync_scripts() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone local grpc_hosts grpc_gce_script_root @@ -600,7 +600,7 @@ grpc_sync_images() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone local grpc_hosts @@ -645,7 +645,7 @@ _grpc_show_servers_args() { # Shows the grpc servers on the GCE instance grpc_show_servers() { # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by _grpc_show_servers local host @@ -683,7 +683,7 @@ grpc_build_proto_bins() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by _grpc_build_proto_bins_args local host @@ -742,7 +742,7 @@ _grpc_launch_servers_args() { # If no servers are specified, it launches all known servers grpc_launch_servers() { # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by _grpc_launch_servers_args local host servers @@ -825,7 +825,7 @@ grpc_launch_servers() { grpc_interop_test() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_interop_test_args @@ -874,7 +874,7 @@ grpc_interop_test() { grpc_cloud_prod_test() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_cloud_prod_test_args @@ -920,7 +920,7 @@ grpc_cloud_prod_test() { grpc_cloud_prod_auth_test() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_cloud_prod_test_args From 0d949f5decbf2d98d5a9c9731292379d996566ef Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 18 Mar 2015 11:44:38 -0700 Subject: [PATCH 4/4] Updates the structure and name of the tar.gz archive - the name includes a version - the top-level directory of the tar has the same name as the archive --- tools/dockerfile/grpc_dist_proto/Dockerfile | 17 ++++++++++------- tools/gce_setup/grpc_docker.sh | 8 ++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tools/dockerfile/grpc_dist_proto/Dockerfile b/tools/dockerfile/grpc_dist_proto/Dockerfile index a15b7ca4a61..b4ed3b6035d 100644 --- a/tools/dockerfile/grpc_dist_proto/Dockerfile +++ b/tools/dockerfile/grpc_dist_proto/Dockerfile @@ -30,6 +30,9 @@ # Dockerfile to build protoc and plugins for inclusion in a release. FROM grpc/base +# Add the file containing the gRPC version +ADD version.txt version.txt + # Install tools needed for building protoc. RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev @@ -56,16 +59,16 @@ WORKDIR /var/local/git/grpc RUN LDFLAGS=-static make plugins # Create an archive containing all the generated binaries. -RUN mkdir /tmp/proto_bins_root -RUN cp -v bins/opt/* /tmp/proto_bins_root -RUN cp -v /tmp/protoc_static/bin/protoc /tmp/proto_bins_root -RUN cd /tmp/proto_bins_root && \ - tar -czf /tmp/proto-bins-linux-$(uname -m).tar.gz * +RUN mkdir /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m) +RUN cp -v bins/opt/* /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m) +RUN cp -v /tmp/protoc_static/bin/protoc /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m) +RUN cd /tmp && \ + tar -czf proto-bins_$(cat /version.txt)_linux-$(uname -m).tar.gz proto-bins_$(cat /version.txt)_linux-$(uname -m) # List the tar contents: provides a way to visually confirm that the contents # are correct. -RUN echo 'proto-bins-linux-tar-$(uname -m) contents:' && \ - tar -ztf /tmp/proto-bins-linux-$(uname -m).tar.gz +RUN echo 'proto-bins_$(cat /version.txt)_linux-tar-$(uname -m) contents:' && \ + tar -ztf /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m).tar.gz diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 7ffd5b2203f..88735fe4507 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -698,10 +698,10 @@ grpc_build_proto_bins() { local label='dist_proto' grpc_update_image -- -h $host $label || return 1 - # run a command to copy the generated output to the local machine + # run a command to copy the generated archive to the docker host local docker_prefix='sudo docker run -v /tmp:/tmp/proto_bins_out' - local tar_name='proto-bins-linux-x86_64.tar.gz' - local cp_cmd="cp /tmp/$tar_name /tmp/proto_bins_out" + local tar_name='proto-bins*.tar.gz' + local cp_cmd="/bin/bash -c 'cp -v /tmp/$tar_name /tmp/proto_bins_out'" local cmd="$docker_prefix grpc/$label $cp_cmd" local ssh_cmd="bash -l -c \"$cmd\"" echo "will run:" @@ -711,7 +711,7 @@ grpc_build_proto_bins() { # copy the tar.gz locally local rmt_tar="$host:/tmp/$tar_name" - local local_copy="$(pwd)/$tar_name" + local local_copy="$(pwd)" gcloud compute copy-files $rmt_tar $local_copy $project_opt $zone_opt || return 1 }