Merge pull request #146 from tbetbetbe/grpc_tools_update_docker_build_pull_from_github

Grpc tools update docker build pull from GitHub

tools/dockerfile/grpc_base/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y \
   libc6 \
   libc6-dbg \
   libc6-dev \
+  libgtest-dev \
   libtool \
   make \
   strace \
@@ -34,23 +35,13 @@ ENV CLOUD_SDK /google-cloud-sdk
 RUN $CLOUD_SDK/install.sh --usage-reporting=true --path-update=true --bash-completion=true --rc-path=/.bashrc --disable-installation-options
 ENV PATH $CLOUD_SDK/bin:$PATH
 
-# Install gcompute-tools to allow access to private git-on-borg repos
+# Install a GitHub SSH service credential that gives access to the GitHub repo while it's private
-RUN git clone https://gerrit.googlesource.com/gcompute-tools /var/local/git/gcompute-tools
+# TODO: remove this once the repo is public
-
+ADD .ssh .ssh
-# Start the daemon that allows access to private git-on-borg repos
+RUN chmod 600 .ssh/github.rsa
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+RUN mkdir -p $HOME/.ssh && echo 'Host github.com' > $HOME/.ssh/config
-
+RUN echo "    IdentityFile /.ssh/github.rsa" >> $HOME/.ssh/config
-# Install the grpc-tools scripts dir from git
+RUN echo 'StrictHostKeyChecking no' >> $HOME/.ssh/config
-RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-tools /var/local/git/grpc-tools
-
-# Install the grpc-protobuf dir that has the protoc patch
-RUN git clone https://team.googlesource.com/one-platform-grpc-team/protobuf /var/local/git/protobuf
-
-# Install the patched version of protoc
-RUN cd /var/local/git/protobuf && \
-  ./autogen.sh && \
-  ./configure --prefix=/usr && \
-  make && make check && make install && make clean
 
 # Define the default command.
 CMD ["bash"]

tools/dockerfile/grpc_cxx/Dockerfile

@@ -1,15 +1,18 @@
 # Dockerfile for gRPC C++
 FROM grpc/base
 
-# Start the daemon that allows access to the protected git-on-borg repos
+# Get the source from GitHub
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+RUN git clone git@github.com:google/grpc.git /var/local/git/grpc
-
+RUN cd /var/local/git/grpc && \
-RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc
+  git pull --recurse-submodules && \
-RUN cd /var/local/git/grpc \
+  git submodule update --init --recursive
-  && git pull --recurse-submodules \
-  && git submodule update --init --recursive
 
+# Build the protobuf library; then the C core.
+RUN cd /var/local/git/grpc/third_party/protobuf && \
+  ./autogen.sh && \
+  ./configure --prefix=/usr && \
+  make -j12 && make check && make install && make clean
 RUN make install -C /var/local/git/grpc
 
 # Define the default command.
-CMD ["bash"]
+CMD ["bash"]

tools/dockerfile/grpc_php/Dockerfile

@@ -1,9 +1,6 @@
 # Dockerfile for gRPC PHP
 FROM grpc/php_base
 
-# Start the daemon that allows access to the protected git-on-borg repos
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
-
 RUN cd /var/local/git/grpc \
   && git pull --recurse-submodules \
   && git submodule update --init --recursive
@@ -15,4 +12,4 @@ RUN cd /var/local/git/grpc/src/php/ext/grpc && git pull && phpize
 # Build the grpc PHP extension
 RUN cd /var/local/git/grpc/src/php/ext/grpc \
   && ./configure \
-  && make
+  && make

tools/dockerfile/grpc_php_base/Dockerfile

@@ -43,9 +43,10 @@ RUN cd /var/local \
   && tar -xf php-5.5.17.tar.gz \
   && cd php-5.5.17 \
   && ./configure --with-zlib=/usr --with-libxml-dir=ext/libxml \
-  && make && make install
+  && make -j12 && make install
 
 # Start the daemon that allows access to the protected git-on-borg repos
+RUN git clone https://gerrit.googlesource.com/gcompute-tools /var/local/git/gcompute-tools
 RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
 
 # Download the patched PHP protobuf so that PHP gRPC clients can be generated
@@ -64,6 +65,18 @@ ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/s
 # rake: a ruby version of make used to build the PHP Protobuf extension
 RUN rvm all do gem install ronn rake
 
+# Get the source from GitHub, this gets the protobuf library as well
+RUN git clone git@github.com:google/grpc.git /var/local/git/grpc
+RUN cd /var/local/git/grpc && \
+  git pull --recurse-submodules && \
+  git submodule update --init --recursive
+
+# Build and install the protobuf library
+RUN cd /var/local/git/grpc/third_party/protobuf && \
+  ./autogen.sh && \
+  ./configure --prefix=/usr && \
+  make -j12 && make check && make install && make clean
+
 # Install the patched PHP protobuf so that PHP gRPC clients can be generated
 # from proto3 schemas.
 RUN cd /var/local/git/protobuf-php \
@@ -75,10 +88,7 @@ RUN wget https://phar.phpunit.de/phpunit.phar \
   && chmod +x phpunit.phar \
   && mv phpunit.phar /usr/local/bin/phpunit
 
-RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc
+# Build the C core
-RUN cd /var/local/git/grpc \
-  && git submodule update --init --recursive
-
 RUN make static_c shared_c -j12 -C /var/local/git/grpc
 
 # Define the default command.

tools/dockerfile/grpc_ruby/Dockerfile

@@ -1,19 +1,14 @@
 # Dockerfile for gRPC Ruby
 FROM grpc/ruby_base
 
-# Start the daemon that allows access to the protected git-on-borg repos
+# Build the C libary
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
-
 RUN cd /var/local/git/grpc \
   && git pull --recurse-submodules \
   && git submodule update --init --recursive
 
+# Build the C core.
 RUN make install_c -C /var/local/git/grpc
 
-# Install the grpc gem locally with its dependencies and build the extension.
-RUN /bin/bash -l -c 'cd /var/local/git/beefcake && bundle && gem build beefcake.gemspec && gem install beefcake'
-RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake compile:grpc && gem build grpc.gemspec && gem install grpc'
-
 # TODO add a command to run the unittest tests when the bug below is fixed
 # - the tests fail due to an error in the C threading library:
 #   they fail with 'ruby: __pthread_mutex_cond_lock_adjust for unknown reasons' at the end of a testcase

tools/dockerfile/grpc_ruby_base/Dockerfile

@@ -31,15 +31,6 @@ RUN apt-get update && apt-get install -y \
     sqlite3 \
     zlib1g-dev
 
-
-# Start the daemon that allows access to the protected git-on-borg repos
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
-
-# Download the patched Ruby protobuf (beefcake) so that Ruby gRPC clients can
-# be generated from proto3 schemas.
-RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-ruby-beefcake \
-              /var/local/git/beefcake
-
 # Install RVM, use this to install ruby
 RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3  # Needed for RVM
 RUN /bin/bash -l -c "curl -L get.rvm.io | bash -s stable"
@@ -52,8 +43,17 @@ RUN /bin/bash -l -c "echo 'source /home/grpc_ruby/.rvm/scripts/rvm' >> ~/.bashrc
 RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
 RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
 
-RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc
+# Get the source from GitHub
-RUN cd /var/local/git/grpc \
+RUN git clone git@github.com:google/grpc.git /var/local/git/grpc
-  && git submodule update --init --recursive
+RUN cd /var/local/git/grpc && \
+  git pull --recurse-submodules && \
+  git submodule update --init --recursive
+
+# Build and install the protobuf library
+RUN cd /var/local/git/grpc/third_party/protobuf && \
+  ./autogen.sh && \
+  ./configure --prefix=/usr && \
+  make -j12 && make check && make install && make clean
 
-RUN make static_c shared_c -C /var/local/git/grpc
+# Build the C core
+RUN make static_c shared_c -j12 -C /var/local/git/grpc

tools/gce_setup/shared_startup_funcs.sh

@@ -388,7 +388,7 @@ grpc_docker_pull_known() {
 # grpc_dockerfile_install "grpc/image" /var/local/dockerfile/grpc_image
 grpc_dockerfile_install() {
   local image_label=$1
-  [[ -n $image_label ]] || { echo "missing arg: image_label" >&2; return 1; }
+  [[ -n $image_label ]] || { echo "$FUNCNAME: missing arg: image_label" >&2; return 1; }
   local docker_img_url=0.0.0.0:5000/$image_label
 
   local dockerfile_dir=$2
@@ -400,19 +400,25 @@ grpc_dockerfile_install() {
   [[ $cache == "cache=1" ]] && { cache_opt=''; }
   [[ $cache == "cache=true" ]] && { cache_opt=''; }
 
-  [[ -d $dockerfile_dir ]] || { echo "not a valid dir: $dockerfile_dir"; return 1; }
+  [[ -d $dockerfile_dir ]] || { echo "$FUNCNAME: not a valid dir: $dockerfile_dir"; return 1; }
+
+  # For grpc/base, sync the ssh key into the  .ssh dir in the dockerfile context
+
+  [[ $image_label == "grpc/base" ]] && {
+    grpc_docker_sync_github_key $dockerfile_dir/.ssh || return 1;
+  }
 
   # TODO(temiola): maybe make cache/no-cache a func option?
   sudo docker build $cache_opt -t $image_label $dockerfile_dir || {
-    echo "docker op error: build of $image_label <- $dockerfile_dir"
+    echo "$FUNCNAME:: build of $image_label <- $dockerfile_dir"
     return 1
   }
   sudo docker tag $image_label $docker_img_url || {
-    echo "docker op error: tag of $docker_img_url"
+    echo "$FUNCNAME: failed to tag $docker_img_url as $image_label"
     return 1
   }
   sudo docker push $docker_img_url || {
-    echo "docker op error: push of $docker_img_url"
+    echo "$FUNCNAME: failed to push $docker_img_url"
     return 1
   }
 }
@@ -428,3 +434,31 @@ grpc_dockerfile_install() {
 grpc_dockerfile_refresh() {
   grpc_dockerfile_install "$@"
 }
+
+# grpc_docker_sync_github_key.
+#
+# Copies the docker github key from GCS to the target dir
+#
+# call-seq:
+#   grpc_docker_sync_github_key <target_dir>
+grpc_docker_sync_github_key() {
+  local target_dir=$1
+  [[ -n $target_dir ]] || { echo "$FUNCNAME: missing arg: target_dir" >&2; return 1; }
+
+  # determine the admin root; the parent of the dockerfile root,
+  local gs_dockerfile_root=$(load_metadata "attributes/gs_dockerfile_root")
+  [[ -n $gs_dockerfile_root ]] || {
+    echo "$FUNCNAME: missing metadata: gs_dockerfile_root" >&2
+    return 1
+  }
+  local gcs_admin_root=$(dirname $gs_dockerfile_root)
+
+  # cp the file from gsutil to a known local area
+  local gcs_key_path=$gcs_admin_root/github/ssh_key
+  local local_key_path=$target_dir/github.rsa
+  mkdir -p $target_dir || {
+    echo "$FUNCNAME: could not create dir: $target_dir" 1>&2
+    return 1
+  }
+  gsutil cp $src $gcs_key_path $local_key_path
+}