Merge https://github.com/grpc/grpc into cg-timeout

doc/python/sphinx/grpc_observability.rst

@@ -0,0 +1,7 @@
+gRPC Python Observability
+====================
+
+Module Contents
+---------------
+
+.. automodule:: grpc_observability

gRPC-C++.podspec

@@ -80,6 +80,14 @@ Pod::Spec.new do |s|
 
   s.header_mappings_dir = 'include/grpcpp'
 
+  # Exposes the privacy manifest. Depended on by any subspecs containing
+  # non-interface files.
+  s.subspec 'Privacy' do |ss|
+    ss.resource_bundles = {
+      s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+    }
+  end
+
   s.subspec 'Interface' do |ss|
     ss.header_mappings_dir = 'include/grpcpp'
 
@@ -214,6 +222,7 @@ Pod::Spec.new do |s|
 
   s.subspec 'Implementation' do |ss|
     ss.header_mappings_dir = '.'
+    ss.dependency "#{s.name}/Privacy", version
     ss.dependency "#{s.name}/Interface", version
     ss.dependency 'gRPC-Core', version
     abseil_version = '1.20230802.0'
@@ -2738,6 +2747,7 @@ Pod::Spec.new do |s|
     ss.header_mappings_dir = '.'
     ss.dependency "#{s.name}/Cronet-Interface", version
     ss.dependency "#{s.name}/Implementation", version
+    ss.dependency "#{s.name}/Privacy", version
 
     ss.dependency 'gRPC-Core/Cronet-Implementation', version
 

gRPC-Core.podspec

@@ -94,6 +94,14 @@ Pod::Spec.new do |s|
   s.compiler_flags = '-DGRPC_ARES=0 -Wno-comma'
   s.libraries = 'c++'
 
+  # Exposes the privacy manifest. Depended on by any subspecs containing
+  # non-interface files.
+  s.subspec 'Privacy' do |ss|
+    ss.resource_bundles = {
+      s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+    }
+  end
+
   # Like many other C libraries, gRPC-Core has its public headers under `include/<libname>/` and its
   # sources and private headers in other directories outside `include/`. Cocoapods' linter doesn't
   # allow any header to be listed outside the `header_mappings_dir` (even though doing so works in
@@ -185,6 +193,7 @@ Pod::Spec.new do |s|
     ss.header_mappings_dir = '.'
     ss.libraries = 'z'
     ss.dependency "#{s.name}/Interface", version
+    ss.dependency "#{s.name}/Privacy", version
     ss.dependency 'BoringSSL-GRPC', '0.0.31'
     ss.dependency 'abseil/algorithm/container', abseil_version
     ss.dependency 'abseil/base/base', abseil_version
@@ -3494,6 +3503,7 @@ Pod::Spec.new do |s|
 
     ss.dependency "#{s.name}/Interface", version
     ss.dependency "#{s.name}/Implementation", version
+    ss.dependency "#{s.name}/Privacy", version
     ss.dependency "#{s.name}/Cronet-Interface", version
 
     ss.source_files = 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc',

gRPC.podspec

@@ -32,8 +32,6 @@ Pod::Spec.new do |s|
     :tag => "v#{version}",
   }
 
-  s.resource = 'src/objective-c/PrivacyInfo.xcprivacy'
-
   name = 'GRPCClient'
   s.module_name = name
   s.header_dir = name
@@ -52,6 +50,14 @@ Pod::Spec.new do |s|
   s.tvos.deployment_target = '12.0'
   s.watchos.deployment_target = '6.0'
 
+  # Exposes the privacy manifest. Depended on by any subspecs containing
+  # non-interface files.
+  s.subspec 'Privacy' do |ss|
+    ss.resource_bundles = {
+      s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+    }
+  end
+
   s.subspec 'Interface-Legacy' do |ss|
     ss.header_mappings_dir = 'src/objective-c/GRPCClient'
 
@@ -72,7 +78,7 @@ Pod::Spec.new do |s|
                       "src/objective-c/GRPCClient/GRPCTypes.h",
                       "src/objective-c/GRPCClient/GRPCTypes.mm"
     ss.dependency "gRPC-RxLibrary/Interface", version
-
+    ss.dependency "#{s.name}/Privacy", version
     s.ios.deployment_target = '10.0'
     s.osx.deployment_target = '10.12'
     s.tvos.deployment_target = '12.0'
@@ -107,7 +113,7 @@ Pod::Spec.new do |s|
                       'src/objective-c/GRPCClient/version.h'
 
     ss.dependency "#{s.name}/Interface-Legacy", version
-
+    ss.dependency "#{s.name}/Privacy", version
     s.ios.deployment_target = '10.0'
     s.osx.deployment_target = '10.12'
     s.tvos.deployment_target = '12.0'
@@ -141,6 +147,7 @@ Pod::Spec.new do |s|
 
     ss.dependency "#{s.name}/Interface-Legacy", version
     ss.dependency "#{s.name}/Interface", version
+    ss.dependency "#{s.name}/Privacy", version
     ss.dependency 'gRPC-Core', version
     ss.dependency 'gRPC-RxLibrary', version
 
@@ -157,6 +164,7 @@ Pod::Spec.new do |s|
                       'src/objective-c/GRPCClient/GRPCCall+Cronet.mm',
                       'src/objective-c/GRPCClient/private/GRPCCore/GRPCCoreCronet/*.{h,mm}'
     ss.dependency "#{s.name}/GRPCCore", version
+    ss.dependency "#{s.name}/Privacy", version
     ss.dependency 'gRPC-Core/Cronet-Implementation', version
     ss.dependency 'CronetFramework'
 

src/core/lib/experiments/experiments.yaml

@@ -115,7 +115,7 @@
 - name: multiping
   description:
     Allow more than one ping to be in flight at a time by default.
-  expiry: 2024/01/15
+  expiry: 2024/06/15
   owner: ctiller@google.com
   test_tags: [flow_control_test]
 - name: peer_state_based_framing
@@ -231,7 +231,7 @@
   test_tags: []
 - name: unconstrained_max_quota_buffer_size
   description: Discard the cap on the max free pool size for one memory allocator
-  expiry: 2024/02/01
+  expiry: 2024/09/01
   owner: ctiller@google.com
   test_tags: [resource_quota_test]
 - name: v3_backend_metric_filter
@@ -261,7 +261,7 @@
 - name: work_serializer_clears_time_cache
   description:
     Have the work serializer clear the time cache when it dispatches work.
-  expiry: 2024/01/01
+  expiry: 2024/04/01
   owner: ctiller@google.com
   test_tags: []
 - name: work_serializer_dispatch

src/objective-c/BoringSSL-GRPC.podspec

@@ -122,6 +122,11 @@ Pod::Spec.new do |s|
   end
   s.subspec 'Implementation' do |ss|
     ss.header_mappings_dir = 'src'
+
+    ss.resource_bundles = {
+      s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+    }
+
     ss.source_files = 'src/ssl/*.{h,c,cc}',
                       'src/ssl/**/*.{h,c,cc}',
                       'src/crypto/*.{h,c,cc}',

src/objective-c/PrivacyInfo.xcprivacy

@@ -18,6 +18,14 @@
 				<string>C617.1</string>
 			</array>
 		</dict>
+		<dict>
+			<key>NSPrivacyAccessedAPIType</key>
+			<string>NSPrivacyAccessedAPICategorySystemBootTime</string>
+			<key>NSPrivacyAccessedAPITypeReasons</key>
+			<array>
+				<string>35F9.1</string>
+			</array>
+		</dict>
 	</array>
 </dict>
 </plist>

src/python/grpcio_observability/README.rst

@@ -1,2 +1,91 @@
-### TODO(xuanwn)
+gRPC Python Observability
-* Fill in content.
+===========
+Package for gRPC Python Observability.
+
+More details can be found in `OpenTelemetry Metrics gRFC <https://github.com/grpc/proposal/blob/master/A66-otel-stats.md#opentelemetry-metrics>`_.
+
+How gRPC Python Observability Works
+-------------------------
+
+gRPC Python is a wrapper layer built upon the gRPC Core (written in C/C++). Most of telemetry data
+is collected at core layer and then exported to Python layer. To optimize performance and reduce
+the overhead of acquiring the GIL too frequently, telemetry data is initially cached at the Core layer
+and then exported to the Python layer in batches.
+
+Note that while this approach enhances efficiency, it will introduce a slight delay between the
+time the data is collected and the time it becomes available through Python exporters.
+
+
+Supported Python Versions
+-------------------------
+Python >= 3.7
+
+Installation
+------------
+
+Currently gRPC Python Observability is **only available for Linux**.
+
+Installing From PyPI
+~~~~~~~~~~~~~~~~~~~~
+
+::
+
+  $ pip install grpcio-observability
+
+
+Installing From Source
+~~~~~~~~~~~~~~~~~~~~~~
+
+Building from source requires that you have the Python headers (usually a
+package named :code:`python-dev`) and Cython installed. It further requires a
+GCC-like compiler to go smoothly; you can probably get it to work without
+GCC-like stuff, but you may end up having a bad time.
+
+::
+
+  $ export REPO_ROOT=grpc  # REPO_ROOT can be any directory of your choice
+  $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc $REPO_ROOT
+  $ cd $REPO_ROOT
+  $ git submodule update --init
+
+  $ cd src/python/grpcio_observability
+  $ python -m make_grpcio_observability
+
+  # For the next command do `sudo pip install` if you get permission-denied errors
+  $ GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+
+
+Dependencies
+-------------------------
+gRPC Python Observability Depends on the following packages:
+
+::
+
+  grpcio
+  opentelemetry-sdk==1.21.0
+  opentelemetry-api==1.21.0
+
+
+Usage
+-----
+
+You can find example usage in `Python example folder <https://github.com/grpc/grpc/tree/master/examples/python/observability>`_.
+
+We also provide several environment variables to help you optimize gRPC python observability for your particular use.
+
+1. GRPC_PYTHON_CENSUS_EXPORT_BATCH_INTERVAL
+    * This controls how frequently telemetry data collected within gRPC Core is sent to Python layer.
+    * Default value is 0.5 (Seconds).
+
+2. GRPC_PYTHON_CENSUS_MAX_EXPORT_BUFFER_SIZE
+    * This controls the maximum number of telemetry data items that can be held in the buffer within gRPC Core before they are sent to Python.
+    * Default value is 10,000.
+
+3. GRPC_PYTHON_CENSUS_EXPORT_THRESHOLD
+    * This setting acts as a trigger: When the buffer in gRPC Core reaches a certain percentage of its capacity, the telemetry data is sent to Python.
+    * Default value is 0.7 (Which means buffer will start export when it's 70% full).
+
+4. GRPC_PYTHON_CENSUS_EXPORT_THREAD_TIMEOUT
+    * This controls the maximum time allowed for the exporting thread (responsible for sending data to Python) to complete.
+    * Main thread will terminate the exporting thread after this timeout.
+    * Default value is 10 (Seconds).

src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py

@@ -59,6 +59,8 @@ GRPC_STATUS_CODE_TO_STRING = {
 class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin):
     """OpenTelemetry based plugin implementation.
 
+    This is class is part of an EXPERIMENTAL API.
+
     Args:
       exporter: Exporter used to export data.
       plugin: OpenTelemetryPlugin to enable.

src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py

@@ -90,14 +90,33 @@ class OpenTelemetryPluginOption(abc.ABC):
 
 # pylint: disable=no-self-use
 class OpenTelemetryPlugin:
-    """Describes a Plugin for OpenTelemetry observability."""
+    """Describes a Plugin for OpenTelemetry observability.
+
+    This is class is part of an EXPERIMENTAL API.
+    """
 
     def get_plugin_options(
         self,
     ) -> Iterable[OpenTelemetryPluginOption]:
+        """
+        This function will be used to get plugin options which are enabled for
+        this OpenTelemetryPlugin instance.
+
+        Returns:
+            An Iterable of class OpenTelemetryPluginOption which will be enabled for
+            this OpenTelemetryPlugin.
+        """
         return []
 
     def get_meter_provider(self) -> Optional[MeterProvider]:
+        """
+        This function will be used to get the MeterProvider for this OpenTelemetryPlugin
+        instance.
+
+        Returns:
+            A MeterProvider which will be used to collect telemetry data, or None which
+            means no metrics will be collected.
+        """
         return None
 
     def target_attribute_filter(

src/python/grpcio_tests/tests/observability/BUILD.bazel

@@ -18,6 +18,11 @@ py_library(
     srcs = ["_test_server.py"],
 )
 
+py_library(
+    name = "_from_observability_import_star",
+    srcs = ["_from_observability_import_star.py"],
+)
+
 py_test(
     name = "_observability_test",
     size = "small",
@@ -45,3 +50,17 @@ py_test(
         "//src/python/grpcio_tests/tests/testing",
     ],
 )
+
+py_test(
+    name = "_observability_api_test",
+    size = "small",
+    srcs = ["_observability_api_test.py"],
+    imports = ["../../"],
+    main = "_observability_api_test.py",
+    deps = [
+        ":_from_observability_import_star",
+        "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_observability/grpc_observability:pyobservability",
+        "//src/python/grpcio_tests/tests/testing",
+    ],
+)

src/python/grpcio_tests/tests/observability/_from_observability_import_star.py

@@ -0,0 +1,25 @@
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+_BEFORE_IMPORT = tuple(globals())
+
+from grpc_observability import *  # pylint: disable=wildcard-import,unused-wildcard-import
+
+_AFTER_IMPORT = tuple(globals())
+
+GRPC_OBSERVABILITY_ELEMENTS = tuple(
+    element
+    for element in _AFTER_IMPORT
+    if element not in _BEFORE_IMPORT and element != "_BEFORE_IMPORT"
+)

src/python/grpcio_tests/tests/observability/_observability_api_test.py

@@ -0,0 +1,37 @@
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test of gRPC Python Observability's application-layer API."""
+
+import logging
+import unittest
+
+from tests.observability import _from_observability_import_star
+
+
+class AllTest(unittest.TestCase):
+    def testAll(self):
+        expected_observability_code_elements = (
+            "OpenTelemetryObservability",
+            "OpenTelemetryPlugin",
+        )
+
+        self.assertCountEqual(
+            expected_observability_code_elements,
+            _from_observability_import_star.GRPC_OBSERVABILITY_ELEMENTS,
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    unittest.main(verbosity=2)

src/python/grpcio_tests/tests/tests.json

@@ -9,6 +9,7 @@
   "tests.health_check._health_servicer_test.HealthServicerTest",
   "tests.interop._insecure_intraop_test.InsecureIntraopTest",
   "tests.interop._secure_intraop_test.SecureIntraopTest",
+  "tests.observability._observability_api_test.AllTest",
   "tests.observability._observability_test.ObservabilityTest",
   "tests.observability._open_telemetry_observability_test.OpenTelemetryObservabilityTest",
   "tests.protoc_plugin._python_plugin_test.ModuleMainTest",

templates/gRPC-C++.podspec.template

@@ -164,6 +164,14 @@
 
     s.header_mappings_dir = 'include/grpcpp'
 
+    # Exposes the privacy manifest. Depended on by any subspecs containing
+    # non-interface files.
+    s.subspec 'Privacy' do |ss|
+      ss.resource_bundles = {
+        s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+      }
+    end
+
     s.subspec 'Interface' do |ss|
       ss.header_mappings_dir = 'include/grpcpp'
 
@@ -172,6 +180,7 @@
 
     s.subspec 'Implementation' do |ss|
       ss.header_mappings_dir = '.'
+      ss.dependency "#{s.name}/Privacy", version
       ss.dependency "#{s.name}/Interface", version
       ss.dependency 'gRPC-Core', version
       abseil_version = '1.20230802.0'
@@ -203,6 +212,7 @@
       ss.header_mappings_dir = '.'
       ss.dependency "#{s.name}/Cronet-Interface", version
       ss.dependency "#{s.name}/Implementation", version
+      ss.dependency "#{s.name}/Privacy", version
 
       ss.dependency 'gRPC-Core/Cronet-Implementation', version
 

templates/gRPC-Core.podspec.template

@@ -170,6 +170,14 @@
     s.compiler_flags = '-DGRPC_ARES=0 -Wno-comma'
     s.libraries = 'c++'
 
+    # Exposes the privacy manifest. Depended on by any subspecs containing
+    # non-interface files.
+    s.subspec 'Privacy' do |ss|
+      ss.resource_bundles = {
+        s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+      }
+    end
+
     # Like many other C libraries, gRPC-Core has its public headers under `include/<libname>/` and its
     # sources and private headers in other directories outside `include/`. Cocoapods' linter doesn't
     # allow any header to be listed outside the `header_mappings_dir` (even though doing so works in
@@ -189,6 +197,7 @@
       ss.header_mappings_dir = '.'
       ss.libraries = 'z'
       ss.dependency "#{s.name}/Interface", version
+      ss.dependency "#{s.name}/Privacy", version
       ss.dependency 'BoringSSL-GRPC', '0.0.31'
       % for abseil_spec in grpc_abseil_specs:
       ss.dependency '${abseil_spec}', abseil_version
@@ -214,6 +223,7 @@
 
       ss.dependency "#{s.name}/Interface", version
       ss.dependency "#{s.name}/Implementation", version
+      ss.dependency "#{s.name}/Privacy", version
       ss.dependency "#{s.name}/Cronet-Interface", version
 
       ss.source_files = ${ruby_multiline_list(grpc_cronet_files, 22)}

templates/gRPC.podspec.template

@@ -34,8 +34,6 @@
       :tag => "v#{version}",
     }
 
-    s.resource = 'src/objective-c/PrivacyInfo.xcprivacy'
-
     name = 'GRPCClient'
     s.module_name = name
     s.header_dir = name
@@ -54,6 +52,14 @@
     s.tvos.deployment_target = '12.0'
     s.watchos.deployment_target = '6.0'
 
+    # Exposes the privacy manifest. Depended on by any subspecs containing
+    # non-interface files.
+    s.subspec 'Privacy' do |ss|
+      ss.resource_bundles = {
+        s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+      }
+    end
+
     s.subspec 'Interface-Legacy' do |ss|
       ss.header_mappings_dir = 'src/objective-c/GRPCClient'
 
@@ -74,7 +80,7 @@
                         "src/objective-c/GRPCClient/GRPCTypes.h",
                         "src/objective-c/GRPCClient/GRPCTypes.mm"
       ss.dependency "gRPC-RxLibrary/Interface", version
-
+      ss.dependency "#{s.name}/Privacy", version
       s.ios.deployment_target = '10.0'
       s.osx.deployment_target = '10.12'
       s.tvos.deployment_target = '12.0'
@@ -109,7 +115,7 @@
                         'src/objective-c/GRPCClient/version.h'
 
       ss.dependency "#{s.name}/Interface-Legacy", version
-
+      ss.dependency "#{s.name}/Privacy", version
       s.ios.deployment_target = '10.0'
       s.osx.deployment_target = '10.12'
       s.tvos.deployment_target = '12.0'
@@ -143,6 +149,7 @@
 
       ss.dependency "#{s.name}/Interface-Legacy", version
       ss.dependency "#{s.name}/Interface", version
+      ss.dependency "#{s.name}/Privacy", version
       ss.dependency 'gRPC-Core', version
       ss.dependency 'gRPC-RxLibrary', version
 
@@ -159,6 +166,7 @@
                         'src/objective-c/GRPCClient/GRPCCall+Cronet.mm',
                         'src/objective-c/GRPCClient/private/GRPCCore/GRPCCoreCronet/*.{h,mm}'
       ss.dependency "#{s.name}/GRPCCore", version
+      ss.dependency "#{s.name}/Privacy", version
       ss.dependency 'gRPC-Core/Cronet-Implementation', version
       ss.dependency 'CronetFramework'
 

templates/src/objective-c/BoringSSL-GRPC.podspec.template

@@ -152,6 +152,11 @@
     end
     s.subspec 'Implementation' do |ss|
       ss.header_mappings_dir = 'src'
+
+      ss.resource_bundles = {
+        s.module_name => 'src/objective-c/PrivacyInfo.xcprivacy'
+      }
+
       ss.source_files = 'src/ssl/*.{h,c,cc}',
                         'src/ssl/**/*.{h,c,cc}',
                         'src/crypto/*.{h,c,cc}',

tools/run_tests/artifacts/artifact_targets.py

@@ -142,7 +142,7 @@ class PythonArtifact:
 
         if self.platform == "macos":
             environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64"
-            environ["GRPC_UNIVERSAL2_REPAIR"] = "true"
+            environ["GRPC_BUILD_MAC"] = "true"
 
         if self.platform == "linux_extra":
             # Crosscompilation build for armv7 (e.g. Raspberry Pi)

tools/run_tests/artifacts/build_artifact_python.sh

@@ -89,6 +89,7 @@ ancillary_package_dir=(
   "src/python/grpcio_reflection/"
   "src/python/grpcio_status/"
   "src/python/grpcio_testing/"
+  "src/python/grpcio_observability/"
 )
 
 # Copy license to ancillary package directories so it will be distributed.
@@ -184,7 +185,7 @@ fix_faulty_universal2_wheel() {
 # This is necessary due to https://github.com/pypa/wheel/issues/406.
 # wheel incorrectly generates a universal2 artifact that only contains
 # x86_64 libraries.
-if [ "$GRPC_UNIVERSAL2_REPAIR" != "" ]; then
+if [ "$GRPC_BUILD_MAC" != "" ]; then
   for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do
     fix_faulty_universal2_wheel "$WHEEL"
   done
@@ -280,4 +281,14 @@ then
   ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_admin/setup.py \
       sdist bdist_wheel
   cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR"
+
+  # Build grpcio_observability source distribution
+  # Skips MacOS since grpcio_observability does not support MacOS.
+  if [ "$GRPC_BUILD_MAC" == "" ]; then
+    "${PYTHON}" src/python/grpcio_observability/make_grpcio_observability.py
+    ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py \
+        sdist bdist_wheel
+    cp -r src/python/grpcio_observability/dist/* "$ARTIFACT_DIR"
+  fi
+
 fi