Fix Python Interop (#27620) (#27703) (#27749)

* WIP. Attempt to fix interop

* Yapf

templates/tools/dockerfile/apt_get_basic_bullseye.include

@@ -0,0 +1,35 @@
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y ${'\\'}
+  autoconf ${'\\'}
+  autotools-dev ${'\\'}
+  build-essential ${'\\'}
+  bzip2 ${'\\'}
+  ccache ${'\\'}
+  curl ${'\\'}
+  dnsutils ${'\\'}
+  gcc ${'\\'}
+  gcc-multilib ${'\\'}
+  git ${'\\'}
+  golang ${'\\'}
+  gyp ${'\\'}
+  lcov ${'\\'}
+  libc6 ${'\\'}
+  libc6-dbg ${'\\'}
+  libc6-dev ${'\\'}
+  libgtest-dev ${'\\'}
+  libtool ${'\\'}
+  make ${'\\'}
+  perl ${'\\'}
+  strace ${'\\'}
+  python3-dev ${'\\'}
+  python3-setuptools ${'\\'}
+  python3-yaml ${'\\'}
+  telnet ${'\\'}
+  unzip ${'\\'}
+  wget ${'\\'}
+  zip && ${'\\'}
+  apt-get clean
+
+#================
+# Build profiling
+RUN apt-get update && apt-get install -y time && apt-get clean

templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  <%include file="../../python_stretch.include"/>
+  <%include file="../../python_bullseye.include"/>
 
-  RUN apt-get update && apt-get -t stable install -y python3.7 python3-all-dev
+  RUN apt-get update && apt-get install -y python3 python3-all-dev python3-pip
-  RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+  RUN ln -s $(which python2) /usr/bin/python

templates/tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  <%include file="../../python_stretch.include"/>
+  <%include file="../../python_bullseye.include"/>
 
-  RUN apt-get update && apt-get -t stable install -y python3.7 python3-all-dev
+  RUN apt-get update && apt-get install -y python3 python3-pip python3-all-dev
-  RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+  RUN ln -s $(which python2) /usr/bin/python

templates/tools/dockerfile/python_build_interop.sh.include

@@ -28,5 +28,5 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# interop tests only run using python3.7 currently (and python build is slow)
+# interop tests only run using python3.9 currently (and python build is slow)
-tools/run_tests/run_tests.py -l python --compiler python3.7 -c opt --build_only
+tools/run_tests/run_tests.py -l python --compiler python3.9 -c opt --build_only

templates/tools/dockerfile/python_bullseye.include

@@ -0,0 +1,6 @@
+FROM debian:bullseye
+  
+<%include file="./apt_get_basic_bullseye.include"/>
+<%include file="./apt_get_python_27.include"/>
+<%include file="./gcp_api_libraries.include"/>
+<%include file="./run_tests_addons.include"/>

tools/dockerfile/interoptest/grpc_interop_python/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:bullseye
   
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -37,13 +37,14 @@ RUN apt-get update && apt-get install -y \
   make \
   perl \
   strace \
-  python-dev \
+  python3-dev \
-  python-setuptools \
+  python3-setuptools \
-  python-yaml \
+  python3-yaml \
   telnet \
   unzip \
   wget \
-  zip && apt-get clean
+  zip && \
+  apt-get clean
 
 #================
 # Build profiling
@@ -56,13 +57,10 @@ RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 # Google Cloud platform API libraries
 RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
-# Add Debian 'buster' repository, we will need it for installing newer versions of python
-RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
 
 
-RUN apt-get update && apt-get -t stable install -y python3.7 python3-all-dev
+RUN apt-get update && apt-get install -y python3 python3-all-dev python3-pip
-RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+RUN ln -s $(which python2) /usr/bin/python

tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh

@@ -28,5 +28,5 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# interop tests only run using python3.7 currently (and python build is slow)
+# interop tests only run using python3.9 currently (and python build is slow)
-tools/run_tests/run_tests.py -l python --compiler python3.7 -c opt --build_only
+tools/run_tests/run_tests.py -l python --compiler python3.9 -c opt --build_only

tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:bullseye
   
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -37,13 +37,14 @@ RUN apt-get update && apt-get install -y \
   make \
   perl \
   strace \
-  python-dev \
+  python3-dev \
-  python-setuptools \
+  python3-setuptools \
-  python-yaml \
+  python3-yaml \
   telnet \
   unzip \
   wget \
-  zip && apt-get clean
+  zip && \
+  apt-get clean
 
 #================
 # Build profiling
@@ -56,13 +57,10 @@ RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 # Google Cloud platform API libraries
 RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
-# Add Debian 'buster' repository, we will need it for installing newer versions of python
-RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
 
 
-RUN apt-get update && apt-get -t stable install -y python3.7 python3-all-dev
+RUN apt-get update && apt-get install -y python3 python3-pip python3-all-dev
-RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+RUN ln -s $(which python2) /usr/bin/python

tools/dockerfile/interoptest/grpc_interop_pythonasyncio/build_interop.sh

@@ -28,5 +28,5 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# interop tests only run using python3.7 currently (and python build is slow)
+# interop tests only run using python3.9 currently (and python build is slow)
-tools/run_tests/run_tests.py -l python --compiler python3.7 -c opt --build_only
+tools/run_tests/run_tests.py -l python --compiler python3.9 -c opt --build_only

tools/run_tests/run_interop_tests.py

@@ -596,6 +596,9 @@ class RubyLanguage:
         return 'ruby'
 
 
+_PYTHON_BINARY = 'py39_native/bin/python'
+
+
 class PythonLanguage:
 
     def __init__(self):
@@ -606,13 +609,13 @@ class PythonLanguage:
 
     def client_cmd(self, args):
         return [
-            'py37_native/bin/python', 'src/python/grpcio_tests/setup.py',
+            _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop',
-            'run_interop', '--client', '--args="{}"'.format(' '.join(args))
+            '--client', '--args="{}"'.format(' '.join(args))
         ]
 
     def client_cmd_http2interop(self, args):
         return [
-            'py37_native/bin/python',
+            _PYTHON_BINARY,
             'src/python/grpcio_tests/tests/http2/negative_http2_client.py',
         ] + args
 
@@ -621,8 +624,8 @@ class PythonLanguage:
 
     def server_cmd(self, args):
         return [
-            'py37_native/bin/python', 'src/python/grpcio_tests/setup.py',
+            _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop',
-            'run_interop', '--server', '--args="{}"'.format(' '.join(args))
+            '--server', '--args="{}"'.format(' '.join(args))
         ]
 
     def global_env(self):
@@ -654,14 +657,13 @@ class PythonAsyncIOLanguage:
 
     def client_cmd(self, args):
         return [
-            'py37_native/bin/python', 'src/python/grpcio_tests/setup.py',
+            _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop',
-            'run_interop', '--use-asyncio', '--client',
+            '--use-asyncio', '--client', '--args="{}"'.format(' '.join(args))
-            '--args="{}"'.format(' '.join(args))
         ]
 
     def client_cmd_http2interop(self, args):
         return [
-            'py37_native/bin/python',
+            _PYTHON_BINARY,
             'src/python/grpcio_tests/tests/http2/negative_http2_client.py',
         ] + args
 
@@ -670,8 +672,8 @@ class PythonAsyncIOLanguage:
 
     def server_cmd(self, args):
         return [
+            _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py',
             'py37_native/bin/python', 'src/python/grpcio_tests/setup.py',
-            'run_interop', '--use-asyncio', '--server',
             '--args="{}"'.format(' '.join(args))
         ]
 

tools/run_tests/run_tests.py

@@ -1405,6 +1405,7 @@ argp.add_argument(
         'python3.6',
         'python3.7',
         'python3.8',
+        'python3.9',
         'pypy',
         'pypy3',
         'python_alpine',