From 8a3ca244b789a36edfcfaa7cf1c5f4a5ece1427a Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 21 Jan 2015 16:09:11 -0800 Subject: [PATCH 1/5] bugfix: corrects a bug in grpc_sync_images --- tools/gce_setup/grpc_docker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index bf776126b5a..314ccb0ca8b 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -412,7 +412,6 @@ grpc_sync_images() { # declare vars local so that they don't pollute the shell environment # where they this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone - # set by grpc_sync_images local grpc_hosts # set the project zone and check that all necessary args are provided @@ -425,7 +424,7 @@ grpc_sync_images() { local host for host in $grpc_hosts do - gce_has_instance $grpc_project $h || return 1; + gce_has_instance $grpc_project $host || return 1; local ssh_cmd="bash -l -c \"$cmd\"" echo "will run:" echo " $ssh_cmd" From 48054c0a19a3cd24a213f7b89ad1c31f3f8c293b Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 12:15:39 -0800 Subject: [PATCH 2/5] Minor bugfixes; adds independent command for support scripts on GCE machines --- tools/gce_setup/grpc_docker.sh | 64 ++++++++++++++++++++++++- tools/gce_setup/shared_startup_funcs.sh | 18 +++++-- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 314ccb0ca8b..a19915594a7 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -86,6 +86,7 @@ grpc_add_docker_user() { } _grpc_update_image_args() { + echo "image_args $@" # default the host, root storage uri and docker file root grpc_gs_root='gs://tmp-grpc-dev/admin/' grpc_dockerfile_root='tools/dockerfile' @@ -95,7 +96,7 @@ _grpc_update_image_args() { # see if -p or -z is used to override the the project or zone local OPTIND local OPTARG - while getopts :r:d:h name + while getopts :r:d:h: name do case $name in d) grpc_dockerfile_root=$OPTARG ;; @@ -261,7 +262,7 @@ _grpc_set_project_and_zone() { local OPTIND local OPTARG local arg_func - while getopts :p:z:f:n name + while getopts :np:z:f: name do case $name in f) declare -F $OPTARG >> /dev/null && { @@ -392,6 +393,65 @@ grpc_interop_test_args() { } } +_grpc_sync_scripts_args() { + grpc_gce_script_root='tools/gce_setup' + + local OPTIND + local OPTARG + while getopts :s: name + do + case $name in + s) grpc_gce_script_root=$OPTARG ;; + :) continue ;; # ignore -s without args, just use the defaults + \?) echo "-$OPTARG: unknown flag; it's ignored" 1>&2; continue ;; + esac + done + shift $((OPTIND-1)) + + [[ -d $grpc_gce_script_root ]] || { + echo "Could not locate gce script dir: $grpc_gce_script_root" 1>&2 + return 1 + } + + [[ $# -lt 1 ]] && { + echo "$FUNCNAME: missing arg: host1 [host2 ... hostN]" 1>&2 + return 1 + } + grpc_hosts="$@" +} + +# Updates the latest version of the support scripts on some hosts. +# +# call-seq; +# grpc_sync_scripts , .. +# +# Updates the GCE docker instance +grpc_sync_scripts() { + _grpc_ensure_gcloud_ssh || return 1; + + # declare vars local so that they don't pollute the shell environment + # where they this func is used. + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone + local grpc_hosts grpc_gce_script_root + + # set the project zone and check that all necessary args are provided + _grpc_set_project_and_zone -f _grpc_sync_scripts_args "$@" || return 1 + + local func_lib="shared_startup_funcs.sh" + local gce_func_lib="/var/local/startup_scripts/$func_lib" + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local host + for host in $grpc_hosts + do + gce_has_instance $grpc_project $host || return 1; + # Update the remote copy of the GCE func library. + local src_func_lib="$grpc_gce_script_root/$func_lib" + local rmt_func_lib="$host:$gce_func_lib" + gcloud compute copy-files $src_func_lib $rmt_func_lib $project_opt $zone_opt || return 1 + done +} + grpc_sync_images_args() { [[ $# -lt 1 ]] && { echo "$FUNCNAME: missing arg: host1 [host2 ... hostN]" 1>&2 diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index 9c747466a91..f9a6aafb095 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -367,7 +367,7 @@ grpc_docker_launch_registry() { grpc_docker_pull_known() { local addr=$1 [[ -n $addr ]] || addr="0.0.0.0:5000" - local known="base cxx php_base php ruby_base ruby java_base java" + local known="base cxx php_base php ruby_base ruby java_base java go" echo "... pulling docker images for '$known'" for i in $known do @@ -402,10 +402,15 @@ grpc_dockerfile_install() { [[ -d $dockerfile_dir ]] || { echo "$FUNCNAME: not a valid dir: $dockerfile_dir"; return 1; } - # For grpc/base, sync the ssh key into the .ssh dir in the dockerfile context - + # For specific base images, sync the ssh key into the .ssh dir in the dockerfile context [[ $image_label == "grpc/base" ]] && { - grpc_docker_sync_github_key $dockerfile_dir/.ssh || return 1; + grpc_docker_sync_github_key $dockerfile_dir/.ssh 'base_ssh_key'|| return 1; + } + [[ $image_label == "grpc/go" ]] && { + grpc_docker_sync_github_key $dockerfile_dir/.ssh 'go_ssh_key'|| return 1; + } + [[ $image_label == "grpc/java_base" ]] && { + grpc_docker_sync_github_key $dockerfile_dir/.ssh 'java_base_ssh_key'|| return 1; } # TODO(temiola): maybe make cache/no-cache a func option? @@ -445,6 +450,9 @@ grpc_docker_sync_github_key() { local target_dir=$1 [[ -n $target_dir ]] || { echo "$FUNCNAME: missing arg: target_dir" >&2; return 1; } + local key_file=$2 + [[ -n $key_file ]] || { echo "$FUNCNAME: missing arg: key_file" >&2; return 1; } + # determine the admin root; the parent of the dockerfile root, local gs_dockerfile_root=$(load_metadata "attributes/gs_dockerfile_root") [[ -n $gs_dockerfile_root ]] || { @@ -454,7 +462,7 @@ grpc_docker_sync_github_key() { local gcs_admin_root=$(dirname $gs_dockerfile_root) # cp the file from gsutil to a known local area - local gcs_key_path=$gcs_admin_root/github/ssh_key + local gcs_key_path=$gcs_admin_root/github/$key_file local local_key_path=$target_dir/github.rsa mkdir -p $target_dir || { echo "$FUNCNAME: could not create dir: $target_dir" 1>&2 From fb6d7fd5720773da1d68f05c2d58cd16bdaf7018 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 12:16:12 -0800 Subject: [PATCH 3/5] Adds Dockerfile for Go --- tools/dockerfile/grpc_go/Dockerfile | 27 +++++++++++++++++++++++++++ tools/dockerfile/grpc_go/README.md | 4 ++++ 2 files changed, 31 insertions(+) create mode 100644 tools/dockerfile/grpc_go/Dockerfile create mode 100644 tools/dockerfile/grpc_go/README.md diff --git a/tools/dockerfile/grpc_go/Dockerfile b/tools/dockerfile/grpc_go/Dockerfile new file mode 100644 index 00000000000..ab463b2a006 --- /dev/null +++ b/tools/dockerfile/grpc_go/Dockerfile @@ -0,0 +1,27 @@ +# Dockerfile for gRPC Go +FROM golang:1.4 + +# Install SSH to that Go source can be pulled securely. +RUN apt-get update && apt-get install -y ssh + +# Install a GitHub SSH service credential that gives access to the GitHub repo while it's private +# +# TODO: remove this once the repo is public +ADD .ssh .ssh +RUN chmod 600 /.ssh/github.rsa +RUN mkdir -p $HOME/.ssh && echo 'Host github.com' > $HOME/.ssh/config +RUN echo " IdentityFile /.ssh/github.rsa" >> $HOME/.ssh/config +RUN echo 'StrictHostKeyChecking no' >> $HOME/.ssh/config + +# Force go get to use the GitHub ssh url instead of https, and use the SSH creds +RUN git config --global url."git@github.com:".insteadOf "https://github.com/" + +# Get the source from GitHub +RUN go get github.com/google/grpc-go + +# Build the interop client and server +RUN cd src/github.com/google/grpc-go/interop/client && go install +RUN cd src/github.com/google/grpc-go/interop/server && go install + +# Specify the default command such that the interop server runs on its known testing port +CMD ["/bin/bash", "-c 'cd src/github.com/google/grpc-go/interop/server && go run server.go --use_tls=true --port=8020'"] diff --git a/tools/dockerfile/grpc_go/README.md b/tools/dockerfile/grpc_go/README.md new file mode 100644 index 00000000000..0d6ad3e391c --- /dev/null +++ b/tools/dockerfile/grpc_go/README.md @@ -0,0 +1,4 @@ +GRPC Go Dockerfile +================== + +Dockerfile for gRPC Go development, testing and deployment. From c8ee19b9052981321992ae0bab7980422211c162 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 17:11:57 -0800 Subject: [PATCH 4/5] Adds a bash func that runs the interop test client command --- tools/gce_setup/grpc_docker.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index a19915594a7..bfa4c808be4 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -634,6 +634,18 @@ grpc_interop_gen_ruby_cmd() { echo $the_cmd } +# constructs the full dockerized Go interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_interop_gen_go_cmd() { + local cmd_prefix="sudo docker run grpc/go bin/bash -c"; + local test_script="cd /go/src/github.com/google/grpc-go/interop/client"; + local test_script+=" && go run client.go --use_tls=true"; + local the_cmd="$cmd_prefix '$test_script $@ 1>&2'"; +} + # constructs the full dockerized java interop test cmd. # # call-seq: @@ -664,4 +676,4 @@ grpc_interop_gen_php_cmd() { } -# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|cxx|nodejs|go +# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|cxx|nodejs From 1a8582f18e1e52151b30845543a2a6b534651fcd Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 17:12:44 -0800 Subject: [PATCH 5/5] removes noisy output when syncing --- tools/gce_setup/shared_startup_funcs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index f9a6aafb095..f1dbca9a2e6 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -371,7 +371,8 @@ grpc_docker_pull_known() { echo "... pulling docker images for '$known'" for i in $known do - sudo docker pull ${addr}/grpc/$i \ + echo "<--- grpc/$i" + sudo docker pull ${addr}/grpc/$i > /dev/null 2>&1 \ && sudo docker tag ${addr}/grpc/$i grpc/$i || { # log and continue echo "docker op error: could not pull ${addr}/grpc/$i"