From befac97048f42f55409bb38a34a75c38cbf34241 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Thu, 22 Sep 2016 17:15:15 -0700
Subject: [PATCH] fixed local cloning of grpc/grpc submodules on docker

---
 test/distrib/cpp/run_distrib_test.sh                       | 6 +++++-
 .../grpc_check_generated_pb_files/check_pb_files.sh        | 6 +++++-
 .../interoptest/grpc_interop_csharp/build_interop.sh       | 6 +++++-
 .../grpc_interop_csharpcoreclr/build_interop.sh            | 6 +++++-
 .../interoptest/grpc_interop_cxx/build_interop.sh          | 6 +++++-
 .../interoptest/grpc_interop_http2/build_interop.sh        | 6 +++++-
 .../interoptest/grpc_interop_node/build_interop.sh         | 6 +++++-
 .../interoptest/grpc_interop_php/build_interop.sh          | 7 ++++++-
 .../interoptest/grpc_interop_php7/build_interop.sh         | 7 ++++++-
 .../interoptest/grpc_interop_python/build_interop.sh       | 6 +++++-
 .../interoptest/grpc_interop_ruby/build_interop.sh         | 7 ++++++-
 .../grpc_interop_stress_csharp/build_interop_stress.sh     | 6 +++++-
 .../grpc_interop_stress_cxx/build_interop_stress.sh        | 6 +++++-
 .../grpc_interop_stress_go/build_interop_stress.sh         | 6 +++++-
 .../grpc_interop_stress_java/build_interop_stress.sh       | 6 +++++-
 .../grpc_interop_stress_node/build_interop_stress.sh       | 6 +++++-
 .../grpc_interop_stress_php/build_interop_stress.sh        | 7 ++++++-
 .../grpc_interop_stress_python/build_interop_stress.sh     | 6 +++++-
 .../grpc_interop_stress_ruby/build_interop_stress.sh       | 7 ++++++-
 tools/run_tests/dockerize/docker_run.sh                    | 6 +++++-
 tools/run_tests/dockerize/docker_run_tests.sh              | 6 +++++-
 21 files changed, 110 insertions(+), 21 deletions(-)

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index bc84b84b8f3..cd4158eb5d9 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -30,9 +30,13 @@
 
 set -ex
 
-git clone --recursive $EXTERNAL_GIT_ROOT
+git clone $EXTERNAL_GIT_ROOT
 cd grpc
 
+# clone submodules
+git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git \
+submodule update --init --reference " EXTERNAL_GIT_ROOT$2 " " $2) }'
+
 cd third_party/protobuf && ./autogen.sh && \
 ./configure && make -j4 && make check && make install && ldconfig
 
diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index 62e41755ec1..fa9fd8fc72e 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -31,10 +31,14 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build grpc cpp plugin for generating grpc pb files
 make grpc_cpp_plugin
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index fd5436c44ff..4ec41b9c30a 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -32,12 +32,16 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index 77cd65cce37..073c4625bc3 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -32,12 +32,16 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 1c0828d23a6..5de62a41c0b 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 make install-certs
 
 # build C++ interop client & server
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 46ddaf929a8..64418bc63de 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -32,7 +32,11 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index 976f55d9ab5..85750ed91c2 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index a84a450221e..ea149049338 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index 261dded2821..0e045631c2f 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index f29c59da8e8..55d1b6f915a 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -32,11 +32,15 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 tools/run_tests/run_tests.py -l python -c opt --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index 97b3860f981..0290de92099 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -32,12 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # build Ruby interop client and server
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index 1f4bf893cce..a4d7feef95f 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # Build C++ metrics client (to query the metrics from csharp stress client)
 make metrics_client -j
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index b67b1a1664a..d704f86496b 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 make install-certs
 
 BUILD_TYPE=${BUILD_TYPE:=opt}
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index 919d885c178..2b3b69be6ee 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -38,7 +38,11 @@ git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 
 # Clone the 'grpc' repo. We just need this for the wrapper scripts under
 # grpc/tools/gcp/stress_tests
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index d4fdfbbac96..99d287b21bf 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -36,7 +36,11 @@ mkdir -p /var/local/git
 git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc-java
 
 # grpc repo (for metrics client and for the stress test wrapper scripts)
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index 976f55d9ab5..85750ed91c2 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index 87262f1d629..997cda6a35a 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 make install-certs
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index e65332f2f30..e3ccd8e00d3 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 tools/run_tests/run_tests.py -l python -c opt --build_only
 
 # Build c++ interop client
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index 1b7567d87a6..be39fb484b2 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -32,12 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # Build Ruby interop client and server
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index f04b1cfb55b..bb875e275e7 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -36,7 +36,11 @@ set -ex
 if [ "$RELATIVE_COPY_PATH" == "" ]
 then
   mkdir -p /var/local/git
-  git clone --recursive "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  # clone gRPC submodules
+  (cd var/local/git/grpc && exec git submodule | awk -v \
+  EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git submodule update --init \
+  --reference " EXTERNAL_GIT_ROOT$2 " " $2) }')
 else
   mkdir -p "/var/local/git/grpc/$RELATIVE_COPY_PATH"
   cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH"
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 8c6143d24f9..55371589605 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -42,7 +42,11 @@ export PATH=$PATH:/usr/bin/llvm-symbolizer
 chown $(whoami) $XDG_CACHE_HOME
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 mkdir -p reports