From 58192e8c9bf1531e2ffafb1cd8cfd885b6825fde Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 15 Jan 2015 12:56:22 -0800 Subject: [PATCH] Simplify the docker image update cmd --- tools/gce_setup/grpc_docker.sh | 164 +++++++++++++----------- tools/gce_setup/shared_startup_funcs.sh | 6 +- 2 files changed, 92 insertions(+), 78 deletions(-) diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index c5d02d77a38..d62f063b53c 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -85,88 +85,98 @@ grpc_add_docker_user() { gcloud compute $project_opt ssh $zone_opt $host --command "$ssh_cmd" } -# Updates a docker image specified in a local dockerfile via the docker -# container GCE instance. -# -# the docker container GCE instance -# - should have been setup using ./new_grpc_docker_instance -# - so will have /var/local/startup_scripts/shared_startup_funcs.sh, a copy of -# ./shared_startup_funcs.sh -# -# grpc_update_image gs://bucket/path/to/dockerfile parent \. -# image_label path/to/docker_dir docker_gce_instance [project] [zone] -grpc_update_image() { - _grpc_ensure_gcloud_ssh || return 1; - local gs_root_uri=$1 - [[ -n $gs_root_uri ]] || { - echo "$FUNCNAME: missing arg: gs_root_uri" 1>&2 - return 1 - } +_grpc_update_image_args() { + # default the host, root storage uri and docker file root + grpc_gs_root='gs://tmp-grpc-dev/admin/' + grpc_dockerfile_root='tools/dockerfile' + grpc_gce_script_root='tools/gce_setup' + host='grpc-docker-builder' - local image_label=$2 - [[ -n $image_label ]] || { - echo "$FUNCNAME: missing arg: host" 1>&2 - return 1 - } + # see if -p or -z is used to override the the project or zone + local OPTIND + local OPTARG + while getopts :r:d:h name + do + case $name in + d) grpc_dockerfile_root=$OPTARG ;; + r) grpc_gs_root=$OPTARG ;; + s) grpc_gce_script_root=$OPTARG ;; + h) host=$OPTARG ;; + :) continue ;; # ignore -r or -d without args, just use the defaults + \?) echo "-$OPTARG: unknown flag; it's ignored" 1>&2; continue ;; + esac + done + shift $((OPTIND-1)) - local docker_dir=$3 - [[ -n $docker_dir ]] || { - echo "$FUNCNAME: missing arg: docker_dir" 1>&2 + [[ -d $grpc_dockerfile_root ]] || { + echo "Could not locate dockerfile root dir: $grpc_dockerfile_root" 1>&2 return 1 } - [[ -d $docker_dir ]] || { - echo "could find directory $docker_dir" 1>&2 + + [[ -d $grpc_gce_script_root ]] || { + echo "Could not locate gce script dir: $grpc_gce_script_root" 1>&2 return 1 } - local docker_parent_dir=$(dirname $docker_dir) - local gce_docker_dir="/var/local/dockerfile/$(basename $docker_dir)" - local host=$4 - [[ -n $host ]] || { - echo "$FUNCNAME: missing arg: host" 1>&2 + # the suffix is required and can't be defaulted + # the suffix has two roles: + # - images are labelled grpc/ + # - the dockerfile is for an image is dockerfile_root/grpc_ + [[ -n $1 ]] && { + label_suffix=$1 + shift + } || { + echo "$FUNCNAME: missing arg: label_suffix (e.g cxx,base,ruby,java_base)" 1>&2 return 1 } +} - local project=$5 - local project_opt='' - [[ -n $project ]] && project_opt=" --project $project" - - local zone=$6 - local zone_opt='' - [[ -n $zone ]] && zone_opt=" --zone $zone" +# Updates a docker image specified in a local dockerfile via the docker +# container GCE instance. +# +# the docker container GCE instance +# - should have been setup using ./new_grpc_docker_instance +# +# There are options for +# +# call-seq: +# grpc_update_image php_base +# grpc_update_image cxx # rebuilds the cxx image +# +grpc_update_image() { + _grpc_ensure_gcloud_ssh || return 1; - local func_lib="/var/local/startup_scripts/shared_startup_funcs.sh" - local ssh_cmd="source $func_lib" + # set up by _grpc_update_args + local host grpc_gs_root grpc_gce_script_root grpc_dockerfile_root label_suffix + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone + _grpc_set_project_and_zone -f _grpc_update_image_args "$@" || return 1 + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local image_label="grpc/$label_suffix" + local docker_dir_basename="grpc_$label_suffix" + local gce_docker_dir="/var/local/dockerfile/${docker_dir_basename}" + + # Set up and run the SSH command that builds the image + local func_lib="shared_startup_funcs.sh" + local gce_func_lib="/var/local/startup_scripts/$func_lib" + local ssh_cmd="source $gce_func_lib" local ssh_cmd+=" && grpc_dockerfile_refresh $image_label $gce_docker_dir" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run - grpc_push_dockerfiles $docker_parent_dir $gs_root_uri || return 1 - gcloud compute $project_opt ssh $zone_opt $host --command "$ssh_cmd" -} - -# gce_has_instance checks if a project contains a named instance -# -# gce_has_instance -gce_has_instance() { - local project=$1 - [[ -n $project ]] || { echo "$FUNCNAME: missing arg: project" 1>&2; return 1; } - local checked_instance=$2 - [[ -n $checked_instance ]] || { - echo "$FUNCNAME: missing arg: checked_instance" 1>&2 - return 1 - } + # Update the remote copy of the GCE func library. + local src_func_lib="$grpc_gce_script_root/$func_lib" + local rmt_func_lib="$host:$gce_func_lib" + gcloud compute copy-files $src_func_lib $rmt_func_lib $project_opt $zone_opt || return 1 - instances=$(gcloud --project $project compute instances list \ - | sed -e 's/ \+/ /g' | cut -d' ' -f 1) - for i in $instances - do - if [[ $i == $checked_instance ]] - then - return 0 - fi - done + # Update the remote version of the docker func. + local src_docker_dir="$grpc_dockerfile_root/$docker_dir_basename" + local rmt_docker_root="$host:/var/local/dockerfile" + gcloud compute copy-files $src_docker_dir $rmt_docker_root $project_opt $zone_opt || return 1 - echo "instance '$checked_instance' not found in compute project $project" 1>&2 - return 1 + gcloud compute $project_opt ssh $zone_opt $host --command "$ssh_cmd" } # gce_find_internal_ip finds the ip address of a instance if it is present in @@ -202,13 +212,19 @@ gce_find_internal_ip() { # - is set to the value gcloud config value for project if that's present # - it defaults to stoked-keyword-656 (the grpc cloud testing project) # - it can be overridden by passing -p -grpc_set_project_and_zone() { +_grpc_set_project_and_zone() { + # can be set to 1 by passing -n in the args dry_run=0 + + # by default; grpc_zone == gcloud config value || asia-east1-a + # - can be assigned via -p in the args grpc_zone=$(gcloud config list compute/zone --format text \ | sed -e 's/ \+/ /g' | cut -d' ' -f 2) # pick a known zone as a default [[ $grpc_zone == 'None' ]] && grpc_zone='asia-east1-a' + # grpc_project == gcloud config value || stoked-keyword-656 + # - can be assigned via -z in the args grpc_project=$(gcloud config list project --format text \ | sed -e 's/ \+/ /g' | cut -d' ' -f 2) # pick an known zone as a default @@ -370,12 +386,12 @@ grpc_update_docker_images() { # declare vars local so that they don't pollute the shell environment # where they this func is used. - local grpc_zone grpc_project dry_run # set by grpc_set_project_and_zone + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by grpc_update_docker_images_args local host # set the project zone and check that all necessary args are provided - grpc_set_project_and_zone -f grpc_update_docker_images_args "$@" || return 1 + _grpc_set_project_and_zone -f grpc_update_docker_images_args "$@" || return 1 gce_has_instance $grpc_project $host || return 1; local func_lib="/var/local/startup_scripts/shared_startup_funcs.sh" @@ -427,12 +443,12 @@ grpc_launch_server_args() { grpc_launch_server() { # declare vars local so that they don't pollute the shell environment # where they this func is used. - local grpc_zone grpc_project dry_run # set by grpc_set_project_and_zone + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by grpc_launch_server_args local docker_label docker_name host grpc_port # set the project zone and check that all necessary args are provided - grpc_set_project_and_zone -f grpc_launch_server_args "$@" || return 1 + _grpc_set_project_and_zone -f grpc_launch_server_args "$@" || return 1 gce_has_instance $grpc_project $host || return 1; cmd="sudo docker run -d --name $docker_name" @@ -491,12 +507,12 @@ grpc_interop_test() { # declare vars local so that they don't pollute the shell environment # where they this func is used. - local grpc_zone grpc_project dry_run # set by grpc_set_project_and_zone + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_interop_test_args local test_case host grpc_gen_test_cmd grpc_server grpc_port # set the project zone and check that all necessary args are provided - grpc_set_project_and_zone -f grpc_interop_test_args "$@" || return 1 + _grpc_set_project_and_zone -f grpc_interop_test_args "$@" || return 1 gce_has_instance $grpc_project $host || return 1; local addr=$(gce_find_internal_ip $grpc_project $grpc_server) diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index 8bd62e213e5..9ea6eca4619 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -423,10 +423,8 @@ grpc_dockerfile_install() { # requires: $2 is a local directory containing a Dockerfile # requires: there is a docker registry running on 5000, e.g, grpc_docker_launch_registry was run # -# invokes pull_dockerfiles to refresh them all from cloud storage, then grpc_dockerfile_install -# -# grpc_dockerfile_refresh "grpc/mylabel" /var/local/dockerfile/dir_containing_my_dockerfile +# call-seq: +# grpc_dockerfile_refresh "grpc/mylabel" /var/local/dockerfile/dir_containing_my_dockerfile grpc_dockerfile_refresh() { - grpc_dockerfile_pull || return 1 grpc_dockerfile_install "$@" }