[Build] Making grpcio_tools ready for upcoming Protobuf change. (#36074)

Protobuf is beginning to rely more on upb, pulling in more C files. Hence, grpcio_tool needs the following changes to absorb this.

- Changing the setup.py to support both C and C++ by removing explicit language=c++. Rather it can choose the right compiler by its extension.
- Adding build_extensions injection to deal with C/C++ option conflict as grpcio does.
- Adding protobuf and upb directories to the build script so that it can find newly added source files in protobuf.

Tested by https://github.com/grpc/grpc/pull/35796

Closes #36074

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36074 from veblush:grpcio-tool-ready 84abce9083
PiperOrigin-RevId: 613975137

tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# distutils: language=c++
 
 from cython.operator cimport dereference
 from libc cimport stdlib

tools/distrib/python/grpcio_tools/protoc_lib_deps.py

@@ -319,7 +319,9 @@ PROTO_FILES=[
 
 CC_INCLUDES=[
     'third_party/abseil-cpp',
+    'third_party/protobuf',
     'third_party/protobuf/src',
+    'third_party/protobuf/upb',
     'third_party/protobuf/third_party/utf8_range'
 ]
 PROTO_INCLUDE='third_party/protobuf/src'

tools/distrib/python/grpcio_tools/setup.py

@@ -126,6 +126,40 @@ class BuildExt(build_ext.build_ext):
             filename = filename[: -len(orig_ext_suffix)] + new_ext_suffix
         return filename
 
+    def build_extensions(self):
+        # This special conditioning is here due to difference of compiler
+        #   behavior in gcc and clang. The clang doesn't take --stdc++11
+        #   flags but gcc does. Since the setuptools of Python only support
+        #   all C or all C++ compilation, the mix of C and C++ will crash.
+        #   *By default*, macOS and FreBSD use clang and Linux use gcc
+        #
+        #   If we are not using a permissive compiler that's OK with being
+        #   passed wrong std flags, swap out compile function by adding a filter
+        #   for it.
+        old_compile = self.compiler._compile
+
+        def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
+            if src.endswith(".c"):
+                extra_postargs = [
+                    arg for arg in extra_postargs if not "-std=c++" in arg
+                ]
+            elif src.endswith(".cc") or src.endswith(".cpp"):
+                extra_postargs = [
+                    arg for arg in extra_postargs if not "-std=c11" in arg
+                ]
+            return old_compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+        self.compiler._compile = new_compile
+
+        try:
+            build_ext.build_ext.build_extensions(self)
+        except Exception as error:
+            formatted_exception = traceback.format_exc()
+            support.diagnose_build_ext_error(self, error, formatted_exception)
+            raise CommandError(
+                "Failed `build_ext` step:\n{}".format(formatted_exception)
+            )
+
 
 # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
@@ -259,7 +293,6 @@ def extension_modules():
             os.path.join("grpc_root", "include"),
         ]
         + CC_INCLUDES,
-        language="c++",
         define_macros=list(DEFINE_MACROS),
         extra_compile_args=list(EXTRA_COMPILE_ARGS),
         extra_link_args=list(EXTRA_LINK_ARGS),

tools/distrib/python/make_grpcio_tools.py

@@ -71,7 +71,9 @@ PROTOBUF_PROTO_PREFIX = "@com_google_protobuf//src/"
 # will be added to include path when building grpcio_tools
 CC_INCLUDES = [
     os.path.join("third_party", "abseil-cpp"),
+    os.path.join("third_party", "protobuf"),
     os.path.join("third_party", "protobuf", "src"),
+    os.path.join("third_party", "protobuf", "upb"),
     os.path.join("third_party", "protobuf", "third_party", "utf8_range"),
 ]
 
@@ -88,6 +90,11 @@ COPY_FILES_SOURCE_TARGET_PAIRS = [
     ("src/compiler", "grpc_root/src/compiler"),
     ("third_party/abseil-cpp/absl", "third_party/abseil-cpp/absl"),
     ("third_party/protobuf/src", "third_party/protobuf/src"),
+    ("third_party/protobuf/upb", "third_party/protobuf/upb"),
+    (
+        "third_party/protobuf/upb_generator",
+        "third_party/protobuf/upb_generator",
+    ),
     (
         "third_party/protobuf/third_party/utf8_range",
         "third_party/protobuf/third_party/utf8_range",
@@ -181,7 +188,7 @@ def _generate_deps_file_content():
     # Collect .cc files (that will be later included in the native extension build)
     cc_files = []
     for name in cc_files_output:
-        if name.endswith(".cc"):
+        if name.endswith(".c") or name.endswith(".cc"):
             filepath = _bazel_name_to_file_path(name)
             if filepath:
                 cc_files.append(filepath)