mirror of https://github.com/grpc/grpc.git
Craig Tiller
1 year ago
commit
3fe1bce51b
42 changed files with 1411 additions and 449 deletions
@ -1,6 +1,7 @@ |
||||
build/ |
||||
include/ |
||||
grpc_root/ |
||||
third_party/ |
||||
*.egg-info/ |
||||
*.c |
||||
*.cpp |
||||
*.egg-info |
||||
*.so |
||||
|
||||
@ -0,0 +1,8 @@ |
||||
graft src/python/grpcio_observability/grpcio_observability.egg-info |
||||
graft grpc_observability |
||||
graft grpc_root |
||||
graft third_party |
||||
include _parallel_compile_patch.py |
||||
include grpc_version.py |
||||
include observability_lib_deps.py |
||||
include README.rst |
||||
@ -0,0 +1,77 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Patches the compile() to allow enable parallel compilation of C/C++. |
||||
|
||||
build_ext has lots of C/C++ files and normally them one by one. |
||||
Enabling parallel build helps a lot. |
||||
""" |
||||
|
||||
import os |
||||
|
||||
try: |
||||
BUILD_EXT_COMPILER_JOBS = int( |
||||
os.environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"] |
||||
) |
||||
except KeyError: |
||||
import multiprocessing |
||||
|
||||
BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count() |
||||
|
||||
|
||||
# monkey-patch for parallel compilation |
||||
# TODO(xuanwn): Use a template for this file. |
||||
def _parallel_compile( |
||||
self, |
||||
sources, |
||||
output_dir=None, |
||||
macros=None, |
||||
include_dirs=None, |
||||
debug=0, |
||||
extra_preargs=None, |
||||
extra_postargs=None, |
||||
depends=None, |
||||
): |
||||
# setup the same way as distutils.ccompiler.CCompiler |
||||
# https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 |
||||
macros, objects, extra_postargs, pp_opts, build = self._setup_compile( |
||||
output_dir, macros, include_dirs, sources, depends, extra_postargs |
||||
) |
||||
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) |
||||
|
||||
def _compile_single_file(obj): |
||||
try: |
||||
src, ext = build[obj] |
||||
except KeyError: |
||||
return |
||||
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) |
||||
|
||||
# run compilation of individual files in parallel |
||||
import multiprocessing.pool |
||||
|
||||
multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( |
||||
_compile_single_file, objects |
||||
) |
||||
return objects |
||||
|
||||
|
||||
def monkeypatch_compile_maybe(): |
||||
""" |
||||
Monkeypatching is dumb, but the build speed gain is worth it. |
||||
After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib. |
||||
""" |
||||
use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "") |
||||
if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib": |
||||
import distutils.ccompiler # pylint: disable=wrong-import-position |
||||
|
||||
distutils.ccompiler.CCompiler.compile = _parallel_compile |
||||
@ -0,0 +1,129 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Helper to read observability config.""" |
||||
|
||||
from dataclasses import dataclass |
||||
from dataclasses import field |
||||
import json |
||||
import os |
||||
from typing import Mapping, Optional |
||||
|
||||
GRPC_GCP_OBSERVABILITY_CONFIG_FILE_ENV = "GRPC_GCP_OBSERVABILITY_CONFIG_FILE" |
||||
GRPC_GCP_OBSERVABILITY_CONFIG_ENV = "GRPC_GCP_OBSERVABILITY_CONFIG" |
||||
|
||||
|
||||
@dataclass |
||||
class GcpObservabilityConfig: |
||||
project_id: str = "" |
||||
stats_enabled: bool = False |
||||
tracing_enabled: bool = False |
||||
labels: Optional[Mapping[str, str]] = field(default_factory=dict) |
||||
sampling_rate: Optional[float] = 0.0 |
||||
|
||||
def load_from_string_content(self, config_contents: str) -> None: |
||||
"""Loads the configuration from a string. |
||||
|
||||
Args: |
||||
config_contents: The configuration string. |
||||
|
||||
Raises: |
||||
ValueError: If the configuration is invalid. |
||||
""" |
||||
try: |
||||
config_json = json.loads(config_contents) |
||||
except json.decoder.JSONDecodeError: |
||||
raise ValueError("Failed to load Json configuration.") |
||||
|
||||
if config_json and not isinstance(config_json, dict): |
||||
raise ValueError("Found invalid configuration.") |
||||
|
||||
self.project_id = config_json.get("project_id", "") |
||||
self.labels = config_json.get("labels", {}) |
||||
self.stats_enabled = "cloud_monitoring" in config_json.keys() |
||||
self.tracing_enabled = "cloud_trace" in config_json.keys() |
||||
tracing_config = config_json.get("cloud_trace", {}) |
||||
self.sampling_rate = tracing_config.get("sampling_rate", 0.0) |
||||
|
||||
|
||||
def read_config() -> GcpObservabilityConfig: |
||||
"""Reads the GCP observability config from the environment variables. |
||||
|
||||
Returns: |
||||
The GCP observability config. |
||||
|
||||
Raises: |
||||
ValueError: If the configuration is invalid. |
||||
""" |
||||
config_contents = _get_gcp_observability_config_contents() |
||||
config = GcpObservabilityConfig() |
||||
config.load_from_string_content(config_contents) |
||||
|
||||
if not config.project_id: |
||||
# Get project ID from GCP environment variables since project ID was not |
||||
# set it in the GCP observability config. |
||||
config.project_id = _get_gcp_project_id_from_env_var() |
||||
if not config.project_id: |
||||
# Could not find project ID from GCP environment variables either. |
||||
raise ValueError("GCP Project ID not found.") |
||||
return config |
||||
|
||||
|
||||
def _get_gcp_project_id_from_env_var() -> Optional[str]: |
||||
"""Gets the project ID from the GCP environment variables. |
||||
|
||||
Returns: |
||||
The project ID, or an empty string if the project ID could not be found. |
||||
""" |
||||
|
||||
project_id = "" |
||||
project_id = os.getenv("GCP_PROJECT") |
||||
if project_id: |
||||
return project_id |
||||
|
||||
project_id = os.getenv("GCLOUD_PROJECT") |
||||
if project_id: |
||||
return project_id |
||||
|
||||
project_id = os.getenv("GOOGLE_CLOUD_PROJECT") |
||||
if project_id: |
||||
return project_id |
||||
|
||||
return project_id |
||||
|
||||
|
||||
def _get_gcp_observability_config_contents() -> str: |
||||
"""Get the contents of the observability config from environment variable or file. |
||||
|
||||
Returns: |
||||
The content from environment variable. |
||||
|
||||
Raises: |
||||
ValueError: If no configuration content was found. |
||||
""" |
||||
|
||||
contents_str = "" |
||||
# First try get config from GRPC_GCP_OBSERVABILITY_CONFIG_FILE_ENV. |
||||
config_path = os.getenv(GRPC_GCP_OBSERVABILITY_CONFIG_FILE_ENV) |
||||
if config_path: |
||||
with open(config_path, "r") as f: |
||||
contents_str = f.read() |
||||
|
||||
# Next, try GRPC_GCP_OBSERVABILITY_CONFIG_ENV env var. |
||||
if not contents_str: |
||||
contents_str = os.getenv(GRPC_GCP_OBSERVABILITY_CONFIG_ENV) |
||||
|
||||
if not contents_str: |
||||
raise ValueError("Configuration content not found.") |
||||
|
||||
return contents_str |
||||
@ -0,0 +1,29 @@ |
||||
// Copyright 2023 gRPC authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include "rpc_encoding.h" |
||||
|
||||
// TODO(xuanwn): Reuse c++ rpc_encoding file.
|
||||
namespace grpc_observability { |
||||
|
||||
constexpr size_t RpcServerStatsEncoding::kRpcServerStatsSize; |
||||
constexpr size_t RpcServerStatsEncoding::kEncodeDecodeFailure; |
||||
constexpr size_t RpcServerStatsEncoding::kVersionIdSize; |
||||
constexpr size_t RpcServerStatsEncoding::kFieldIdSize; |
||||
constexpr size_t RpcServerStatsEncoding::kVersionIdOffset; |
||||
constexpr size_t RpcServerStatsEncoding::kVersionId; |
||||
|
||||
} // namespace grpc_observability
|
||||
@ -0,0 +1,108 @@ |
||||
// Copyright 2023 gRPC authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef GRPC_PYTHON_OPENCENSUS_RPC_ENCODING_H |
||||
#define GRPC_PYTHON_OPENCENSUS_RPC_ENCODING_H |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include <stdint.h> |
||||
#include <string.h> |
||||
|
||||
#include "absl/base/internal/endian.h" |
||||
#include "absl/strings/string_view.h" |
||||
|
||||
namespace grpc_observability { |
||||
|
||||
// TODO(xuanwn): Reuse c++ rpc_encoding file.
|
||||
// RpcServerStatsEncoding encapsulates the logic for encoding and decoding of
|
||||
// rpc server stats messages. Rpc server stats consists of a uint64_t time
|
||||
// value (server latency in nanoseconds).
|
||||
class RpcServerStatsEncoding { |
||||
public: |
||||
// Size of encoded RPC server stats.
|
||||
static constexpr size_t kRpcServerStatsSize = 10; |
||||
// Error value.
|
||||
static constexpr size_t kEncodeDecodeFailure = 0; |
||||
|
||||
// Deserializes rpc server stats from the incoming 'buf' into *time. Returns
|
||||
// number of bytes decoded. If the buffer is of insufficient size (it must be
|
||||
// at least kRpcServerStatsSize bytes) or the encoding version or field ID are
|
||||
// unrecognized, *time will be set to 0 and it will return
|
||||
// kEncodeDecodeFailure. Inlined for performance reasons.
|
||||
static size_t Decode(absl::string_view buf, uint64_t* time) { |
||||
if (buf.size() < kRpcServerStatsSize) { |
||||
*time = 0; |
||||
return kEncodeDecodeFailure; |
||||
} |
||||
|
||||
uint8_t version = buf[kVersionIdOffset]; |
||||
uint32_t fieldID = buf[kServerElapsedTimeOffset]; |
||||
if (version != kVersionId || fieldID != kServerElapsedTimeField) { |
||||
*time = 0; |
||||
return kEncodeDecodeFailure; |
||||
} |
||||
*time = absl::little_endian::Load64( |
||||
&buf[kServerElapsedTimeOffset + kFieldIdSize]); |
||||
return kRpcServerStatsSize; |
||||
} |
||||
|
||||
// Serializes rpc server stats into the provided buffer. It returns the
|
||||
// number of bytes written to the buffer. If the buffer is smaller than
|
||||
// kRpcServerStatsSize bytes it will return kEncodeDecodeFailure. Inlined for
|
||||
// performance reasons.
|
||||
static size_t Encode(uint64_t time, char* buf, size_t buf_size) { |
||||
if (buf_size < kRpcServerStatsSize) { |
||||
return kEncodeDecodeFailure; |
||||
} |
||||
|
||||
buf[kVersionIdOffset] = kVersionId; |
||||
buf[kServerElapsedTimeOffset] = kServerElapsedTimeField; |
||||
absl::little_endian::Store64(&buf[kServerElapsedTimeOffset + kFieldIdSize], |
||||
time); |
||||
return kRpcServerStatsSize; |
||||
} |
||||
|
||||
private: |
||||
// Size of Version ID.
|
||||
static constexpr size_t kVersionIdSize = 1; |
||||
// Size of Field ID.
|
||||
static constexpr size_t kFieldIdSize = 1; |
||||
|
||||
// Offset and value for currently supported version ID.
|
||||
static constexpr size_t kVersionIdOffset = 0; |
||||
static constexpr size_t kVersionId = 0; |
||||
|
||||
enum FieldIdValue { |
||||
kServerElapsedTimeField = 0, |
||||
}; |
||||
|
||||
enum FieldSize { |
||||
kServerElapsedTimeSize = 8, |
||||
}; |
||||
|
||||
enum FieldIdOffset { |
||||
kServerElapsedTimeOffset = kVersionIdSize, |
||||
}; |
||||
|
||||
RpcServerStatsEncoding() = delete; |
||||
RpcServerStatsEncoding(const RpcServerStatsEncoding&) = delete; |
||||
RpcServerStatsEncoding(RpcServerStatsEncoding&&) = delete; |
||||
RpcServerStatsEncoding operator=(const RpcServerStatsEncoding&) = delete; |
||||
RpcServerStatsEncoding operator=(RpcServerStatsEncoding&&) = delete; |
||||
}; |
||||
|
||||
} // namespace grpc_observability
|
||||
|
||||
#endif // GRPC_PYTHON_OPENCENSUS_RPC_ENCODING_H
|
||||
@ -0,0 +1,17 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_observability/grpc_version.py.template`!!! |
||||
|
||||
VERSION = '1.61.0.dev0' |
||||
@ -0,0 +1,215 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import errno |
||||
import os |
||||
import os.path |
||||
import pprint |
||||
import shutil |
||||
import subprocess |
||||
import sys |
||||
import traceback |
||||
|
||||
# the template for the content of observability_lib_deps.py |
||||
DEPS_FILE_CONTENT = """ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# AUTO-GENERATED BY make_grpcio_observability.py! |
||||
CC_FILES={cc_files} |
||||
|
||||
CC_INCLUDES={cc_includes} |
||||
""" |
||||
|
||||
# maps bazel reference to actual path |
||||
BAZEL_REFERENCE_LINK = [ |
||||
("@com_google_absl//", "third_party/abseil-cpp/"), |
||||
("//src", "grpc_root/src"), |
||||
] |
||||
|
||||
ABSL_INCLUDE = (os.path.join("third_party", "abseil-cpp"),) |
||||
|
||||
# will be added to include path when building grpcio_observability |
||||
EXTENSION_INCLUDE_DIRECTORIES = ABSL_INCLUDE |
||||
|
||||
CC_INCLUDES = list(EXTENSION_INCLUDE_DIRECTORIES) |
||||
|
||||
# the target directory is relative to the grpcio_observability package root. |
||||
GRPCIO_OBSERVABILITY_ROOT_PREFIX = "src/python/grpcio_observability/" |
||||
|
||||
# Pairs of (source, target) directories to copy |
||||
# from the grpc repo root to the grpcio_observability build root. |
||||
COPY_FILES_SOURCE_TARGET_PAIRS = [ |
||||
("include", "grpc_root/include"), |
||||
("third_party/abseil-cpp/absl", "third_party/abseil-cpp/absl"), |
||||
("src/core/lib", "grpc_root/src/core/lib"), |
||||
( |
||||
"src/core/ext/filters/client_channel/lb_policy", |
||||
"grpc_root/src/core/ext/filters/client_channel/lb_policy", |
||||
), |
||||
] |
||||
|
||||
# grpc repo root |
||||
GRPC_ROOT = os.path.abspath( |
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") |
||||
) |
||||
|
||||
|
||||
# the file to generate |
||||
GRPC_PYTHON_OBSERVABILITY_LIB_DEPS = os.path.join( |
||||
GRPC_ROOT, |
||||
"src", |
||||
"python", |
||||
"grpcio_observability", |
||||
"observability_lib_deps.py", |
||||
) |
||||
|
||||
# the script to run for getting dependencies |
||||
BAZEL_DEPS = os.path.join( |
||||
GRPC_ROOT, "tools", "distrib", "python", "bazel_deps.sh" |
||||
) |
||||
|
||||
# the bazel target to scrape to get list of sources for the build |
||||
BAZEL_DEPS_QUERIES = [ |
||||
"//src/core:slice", |
||||
] |
||||
|
||||
|
||||
def _bazel_query(query): |
||||
"""Runs 'bazel query' to collect source file info.""" |
||||
print('Running "bazel query %s"' % query) |
||||
output = subprocess.check_output([BAZEL_DEPS, query]) |
||||
return output.decode("ascii").splitlines() |
||||
|
||||
|
||||
def _pretty_print_list(items): |
||||
"""Pretty print python list""" |
||||
formatted = pprint.pformat(items, indent=4) |
||||
# add newline after opening bracket (and fix indent of the next line) |
||||
if formatted.startswith("["): |
||||
formatted = formatted[0] + "\n " + formatted[1:] |
||||
# add newline before closing bracket |
||||
if formatted.endswith("]"): |
||||
formatted = formatted[:-1] + "\n" + formatted[-1] |
||||
return formatted |
||||
|
||||
|
||||
def _bazel_name_to_file_path(name): |
||||
"""Transform bazel reference to source file name.""" |
||||
for link in BAZEL_REFERENCE_LINK: |
||||
if name.startswith(link[0]): |
||||
filepath = link[1] + name[len(link[0]) :].replace(":", "/") |
||||
return filepath |
||||
return None |
||||
|
||||
|
||||
def _generate_deps_file_content(): |
||||
"""Returns the data structure with dependencies of protoc as python code.""" |
||||
cc_files_output = [] |
||||
for query in BAZEL_DEPS_QUERIES: |
||||
cc_files_output += _bazel_query(query) |
||||
|
||||
# Collect .cc files (that will be later included in the native extension build) |
||||
cc_files = set() |
||||
for name in cc_files_output: |
||||
if name.endswith(".cc"): |
||||
filepath = _bazel_name_to_file_path(name) |
||||
if filepath: |
||||
cc_files.add(filepath) |
||||
|
||||
deps_file_content = DEPS_FILE_CONTENT.format( |
||||
cc_files=_pretty_print_list(sorted(list(cc_files))), |
||||
cc_includes=_pretty_print_list(CC_INCLUDES), |
||||
) |
||||
return deps_file_content |
||||
|
||||
|
||||
def _copy_source_tree(source, target): |
||||
"""Copies source directory to a given target directory.""" |
||||
print("Copying contents of %s to %s" % (source, target)) |
||||
for source_dir, _, files in os.walk(source): |
||||
target_dir = os.path.abspath( |
||||
os.path.join(target, os.path.relpath(source_dir, source)) |
||||
) |
||||
try: |
||||
os.makedirs(target_dir) |
||||
except OSError as error: |
||||
if error.errno != errno.EEXIST: |
||||
raise |
||||
for relative_file in files: |
||||
source_file = os.path.abspath( |
||||
os.path.join(source_dir, relative_file) |
||||
) |
||||
target_file = os.path.abspath( |
||||
os.path.join(target_dir, relative_file) |
||||
) |
||||
shutil.copyfile(source_file, target_file) |
||||
|
||||
|
||||
def main(): |
||||
os.chdir(GRPC_ROOT) |
||||
|
||||
# Step 1: |
||||
# In order to be able to build the grpcio_observability package, we need the source |
||||
# code for the plugins and its dependencies to be available under the build root of |
||||
# the grpcio_observability package. |
||||
# So we simply copy all the necessary files where the build will expect them to be. |
||||
for source, target in COPY_FILES_SOURCE_TARGET_PAIRS: |
||||
# convert the slashes in the relative path to platform-specific path dividers. |
||||
# All paths are relative to GRPC_ROOT |
||||
source_abs = os.path.join(GRPC_ROOT, os.path.join(*source.split("/"))) |
||||
# for targets, add grpcio_observability root prefix |
||||
target = GRPCIO_OBSERVABILITY_ROOT_PREFIX + target |
||||
target_abs = os.path.join(GRPC_ROOT, os.path.join(*target.split("/"))) |
||||
_copy_source_tree(source_abs, target_abs) |
||||
print( |
||||
"The necessary source files were copied under the grpcio_observability package root." |
||||
) |
||||
|
||||
# Step 2: |
||||
# Extract build metadata from bazel build (by running "bazel query") |
||||
# and populate the observability_lib_deps.py file with python-readable data structure |
||||
# that will be used by grpcio_observability's setup.py. |
||||
try: |
||||
print('Invoking "bazel query" to gather the dependencies.') |
||||
observability_lib_deps_content = _generate_deps_file_content() |
||||
except Exception as error: |
||||
# We allow this script to succeed even if we couldn't get the dependencies, |
||||
# as then we can assume that even without a successful bazel run the |
||||
# dependencies currently in source control are 'good enough'. |
||||
sys.stderr.write("Got non-fatal error:\n") |
||||
traceback.print_exc(file=sys.stderr) |
||||
return |
||||
# If we successfully got the dependencies, truncate and rewrite the deps file. |
||||
with open(GRPC_PYTHON_OBSERVABILITY_LIB_DEPS, "w") as deps_file: |
||||
deps_file.write(observability_lib_deps_content) |
||||
print('File "%s" updated.' % GRPC_PYTHON_OBSERVABILITY_LIB_DEPS) |
||||
print("Done.") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
main() |
||||
@ -0,0 +1,193 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# AUTO-GENERATED BY make_grpcio_observability.py! |
||||
CC_FILES = [ |
||||
"grpc_root/src/core/lib/config/config_vars.cc", |
||||
"grpc_root/src/core/lib/config/config_vars_non_generated.cc", |
||||
"grpc_root/src/core/lib/config/load_config.cc", |
||||
"grpc_root/src/core/lib/debug/trace.cc", |
||||
"grpc_root/src/core/lib/event_engine/thread_local.cc", |
||||
"grpc_root/src/core/lib/gpr/alloc.cc", |
||||
"grpc_root/src/core/lib/gpr/android/log.cc", |
||||
"grpc_root/src/core/lib/gpr/atm.cc", |
||||
"grpc_root/src/core/lib/gpr/iphone/cpu.cc", |
||||
"grpc_root/src/core/lib/gpr/linux/cpu.cc", |
||||
"grpc_root/src/core/lib/gpr/linux/log.cc", |
||||
"grpc_root/src/core/lib/gpr/log.cc", |
||||
"grpc_root/src/core/lib/gpr/msys/tmpfile.cc", |
||||
"grpc_root/src/core/lib/gpr/posix/cpu.cc", |
||||
"grpc_root/src/core/lib/gpr/posix/log.cc", |
||||
"grpc_root/src/core/lib/gpr/posix/string.cc", |
||||
"grpc_root/src/core/lib/gpr/posix/sync.cc", |
||||
"grpc_root/src/core/lib/gpr/posix/time.cc", |
||||
"grpc_root/src/core/lib/gpr/posix/tmpfile.cc", |
||||
"grpc_root/src/core/lib/gpr/string.cc", |
||||
"grpc_root/src/core/lib/gpr/sync.cc", |
||||
"grpc_root/src/core/lib/gpr/sync_abseil.cc", |
||||
"grpc_root/src/core/lib/gpr/time.cc", |
||||
"grpc_root/src/core/lib/gpr/time_precise.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/cpu.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/log.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/string.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/string_util.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/sync.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/time.cc", |
||||
"grpc_root/src/core/lib/gpr/windows/tmpfile.cc", |
||||
"grpc_root/src/core/lib/gpr/wrap_memcpy.cc", |
||||
"grpc_root/src/core/lib/gprpp/crash.cc", |
||||
"grpc_root/src/core/lib/gprpp/examine_stack.cc", |
||||
"grpc_root/src/core/lib/gprpp/fork.cc", |
||||
"grpc_root/src/core/lib/gprpp/host_port.cc", |
||||
"grpc_root/src/core/lib/gprpp/linux/env.cc", |
||||
"grpc_root/src/core/lib/gprpp/mpscq.cc", |
||||
"grpc_root/src/core/lib/gprpp/posix/env.cc", |
||||
"grpc_root/src/core/lib/gprpp/posix/stat.cc", |
||||
"grpc_root/src/core/lib/gprpp/posix/thd.cc", |
||||
"grpc_root/src/core/lib/gprpp/strerror.cc", |
||||
"grpc_root/src/core/lib/gprpp/tchar.cc", |
||||
"grpc_root/src/core/lib/gprpp/time_util.cc", |
||||
"grpc_root/src/core/lib/gprpp/windows/env.cc", |
||||
"grpc_root/src/core/lib/gprpp/windows/stat.cc", |
||||
"grpc_root/src/core/lib/gprpp/windows/thd.cc", |
||||
"grpc_root/src/core/lib/slice/slice.cc", |
||||
"grpc_root/src/core/lib/slice/slice_refcount.cc", |
||||
"grpc_root/src/core/lib/slice/slice_string_helpers.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/cycleclock.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/raw_logging.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/spinlock.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/strerror.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/sysinfo.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/thread_identity.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/throw_delegate.cc", |
||||
"third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc", |
||||
"third_party/abseil-cpp/absl/base/log_severity.cc", |
||||
"third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc", |
||||
"third_party/abseil-cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc", |
||||
"third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc", |
||||
"third_party/abseil-cpp/absl/crc/crc32c.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/cpu_detect.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/crc.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_64.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc", |
||||
"third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc", |
||||
"third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc", |
||||
"third_party/abseil-cpp/absl/debugging/internal/demangle.cc", |
||||
"third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc", |
||||
"third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc", |
||||
"third_party/abseil-cpp/absl/debugging/stacktrace.cc", |
||||
"third_party/abseil-cpp/absl/debugging/symbolize.cc", |
||||
"third_party/abseil-cpp/absl/flags/commandlineflag.cc", |
||||
"third_party/abseil-cpp/absl/flags/flag.cc", |
||||
"third_party/abseil-cpp/absl/flags/internal/commandlineflag.cc", |
||||
"third_party/abseil-cpp/absl/flags/internal/flag.cc", |
||||
"third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.cc", |
||||
"third_party/abseil-cpp/absl/flags/internal/program_name.cc", |
||||
"third_party/abseil-cpp/absl/flags/marshalling.cc", |
||||
"third_party/abseil-cpp/absl/flags/reflection.cc", |
||||
"third_party/abseil-cpp/absl/flags/usage_config.cc", |
||||
"third_party/abseil-cpp/absl/hash/internal/city.cc", |
||||
"third_party/abseil-cpp/absl/hash/internal/hash.cc", |
||||
"third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc", |
||||
"third_party/abseil-cpp/absl/numeric/int128.cc", |
||||
"third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc", |
||||
"third_party/abseil-cpp/absl/random/discrete_distribution.cc", |
||||
"third_party/abseil-cpp/absl/random/gaussian_distribution.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/pool_urbg.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/randen.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/randen_detect.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/randen_hwaes.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/randen_round_keys.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/randen_slow.cc", |
||||
"third_party/abseil-cpp/absl/random/internal/seed_material.cc", |
||||
"third_party/abseil-cpp/absl/random/seed_gen_exception.cc", |
||||
"third_party/abseil-cpp/absl/random/seed_sequences.cc", |
||||
"third_party/abseil-cpp/absl/status/status.cc", |
||||
"third_party/abseil-cpp/absl/status/status_payload_printer.cc", |
||||
"third_party/abseil-cpp/absl/status/statusor.cc", |
||||
"third_party/abseil-cpp/absl/strings/ascii.cc", |
||||
"third_party/abseil-cpp/absl/strings/charconv.cc", |
||||
"third_party/abseil-cpp/absl/strings/cord.cc", |
||||
"third_party/abseil-cpp/absl/strings/cord_analysis.cc", |
||||
"third_party/abseil-cpp/absl/strings/cord_buffer.cc", |
||||
"third_party/abseil-cpp/absl/strings/escaping.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_internal.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cord_rep_ring.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cordz_functions.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cordz_handle.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/cordz_info.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/escaping.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/memutil.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/ostringstream.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/str_format/arg.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/str_format/bind.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/str_format/extension.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/str_format/output.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/str_format/parser.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/stringify_sink.cc", |
||||
"third_party/abseil-cpp/absl/strings/internal/utf8.cc", |
||||
"third_party/abseil-cpp/absl/strings/match.cc", |
||||
"third_party/abseil-cpp/absl/strings/numbers.cc", |
||||
"third_party/abseil-cpp/absl/strings/str_cat.cc", |
||||
"third_party/abseil-cpp/absl/strings/str_replace.cc", |
||||
"third_party/abseil-cpp/absl/strings/str_split.cc", |
||||
"third_party/abseil-cpp/absl/strings/string_view.cc", |
||||
"third_party/abseil-cpp/absl/strings/substitute.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/barrier.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/blocking_counter.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/futex_waiter.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/kernel_timeout.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/pthread_waiter.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/sem_waiter.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/stdcpp_waiter.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/waiter_base.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/internal/win32_waiter.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/mutex.cc", |
||||
"third_party/abseil-cpp/absl/synchronization/notification.cc", |
||||
"third_party/abseil-cpp/absl/time/civil_time.cc", |
||||
"third_party/abseil-cpp/absl/time/clock.cc", |
||||
"third_party/abseil-cpp/absl/time/duration.cc", |
||||
"third_party/abseil-cpp/absl/time/format.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/civil_time_detail.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_fixed.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_if.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_impl.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc", |
||||
"third_party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc", |
||||
"third_party/abseil-cpp/absl/time/time.cc", |
||||
"third_party/abseil-cpp/absl/types/bad_optional_access.cc", |
||||
"third_party/abseil-cpp/absl/types/bad_variant_access.cc", |
||||
] |
||||
|
||||
CC_INCLUDES = ["third_party/abseil-cpp"] |
||||
@ -0,0 +1,297 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import os |
||||
import os.path |
||||
import platform |
||||
import re |
||||
import shlex |
||||
import subprocess |
||||
from subprocess import PIPE |
||||
import sys |
||||
import sysconfig |
||||
|
||||
import pkg_resources |
||||
import setuptools |
||||
from setuptools import Extension |
||||
from setuptools.command import build_ext |
||||
|
||||
PYTHON_STEM = os.path.realpath(os.path.dirname(__file__)) |
||||
README_PATH = os.path.join(PYTHON_STEM, "README.rst") |
||||
|
||||
os.chdir(os.path.dirname(os.path.abspath(__file__))) |
||||
sys.path.insert(0, os.path.abspath(".")) |
||||
|
||||
import _parallel_compile_patch |
||||
import observability_lib_deps |
||||
|
||||
import grpc_version |
||||
|
||||
_parallel_compile_patch.monkeypatch_compile_maybe() |
||||
|
||||
CLASSIFIERS = [ |
||||
"Private :: Do Not Upload", |
||||
"Programming Language :: Python", |
||||
"Programming Language :: Python :: 3", |
||||
"License :: OSI Approved :: Apache Software License", |
||||
] |
||||
|
||||
O11Y_CC_SRCS = [ |
||||
"server_call_tracer.cc", |
||||
"client_call_tracer.cc", |
||||
"observability_util.cc", |
||||
"python_census_context.cc", |
||||
"sampler.cc", |
||||
"rpc_encoding.cc", |
||||
] |
||||
|
||||
|
||||
def _env_bool_value(env_name, default): |
||||
"""Parses a bool option from an environment variable""" |
||||
return os.environ.get(env_name, default).upper() not in ["FALSE", "0", ""] |
||||
|
||||
|
||||
# Environment variable to determine whether or not the Cython extension should |
||||
# *use* Cython or use the generated C files. Note that this requires the C files |
||||
# to have been generated by building first *with* Cython support. |
||||
BUILD_WITH_CYTHON = _env_bool_value("GRPC_PYTHON_BUILD_WITH_CYTHON", "False") |
||||
|
||||
# Export this variable to force building the python extension with a statically linked libstdc++. |
||||
# At least on linux, this is normally not needed as we can build manylinux-compatible wheels on linux just fine |
||||
# without statically linking libstdc++ (which leads to a slight increase in the wheel size). |
||||
# This option is useful when crosscompiling wheels for aarch64 where |
||||
# it's difficult to ensure that the crosscompilation toolchain has a high-enough version |
||||
# of GCC (we require >=5.1) but still uses old-enough libstdc++ symbols. |
||||
# TODO(jtattermusch): remove this workaround once issues with crosscompiler version are resolved. |
||||
BUILD_WITH_STATIC_LIBSTDCXX = _env_bool_value( |
||||
"GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX", "False" |
||||
) |
||||
|
||||
|
||||
def check_linker_need_libatomic(): |
||||
"""Test if linker on system needs libatomic.""" |
||||
code_test = ( |
||||
b"#include <atomic>\n" |
||||
+ b"int main() { return std::atomic<int64_t>{}; }" |
||||
) |
||||
cxx = shlex.split(os.environ.get("CXX", "c++")) |
||||
cpp_test = subprocess.Popen( |
||||
cxx + ["-x", "c++", "-std=c++14", "-"], |
||||
stdin=PIPE, |
||||
stdout=PIPE, |
||||
stderr=PIPE, |
||||
) |
||||
cpp_test.communicate(input=code_test) |
||||
if cpp_test.returncode == 0: |
||||
return False |
||||
# Double-check to see if -latomic actually can solve the problem. |
||||
# https://github.com/grpc/grpc/issues/22491 |
||||
cpp_test = subprocess.Popen( |
||||
cxx + ["-x", "c++", "-std=c++14", "-", "-latomic"], |
||||
stdin=PIPE, |
||||
stdout=PIPE, |
||||
stderr=PIPE, |
||||
) |
||||
cpp_test.communicate(input=code_test) |
||||
return cpp_test.returncode == 0 |
||||
|
||||
|
||||
class BuildExt(build_ext.build_ext): |
||||
"""Custom build_ext command.""" |
||||
|
||||
def get_ext_filename(self, ext_name): |
||||
# since python3.5, python extensions' shared libraries use a suffix that corresponds to the value |
||||
# of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets. |
||||
# E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" |
||||
# When crosscompiling python wheels, we need to be able to override this suffix |
||||
# so that the resulting file name matches the target architecture and we end up with a well-formed |
||||
# wheel. |
||||
filename = build_ext.build_ext.get_ext_filename(self, ext_name) |
||||
orig_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") |
||||
new_ext_suffix = os.getenv("GRPC_PYTHON_OVERRIDE_EXT_SUFFIX") |
||||
if new_ext_suffix and filename.endswith(orig_ext_suffix): |
||||
filename = filename[: -len(orig_ext_suffix)] + new_ext_suffix |
||||
return filename |
||||
|
||||
|
||||
# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are |
||||
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. |
||||
# We use these environment variables to thus get around that without locking |
||||
# ourselves in w.r.t. the multitude of operating systems this ought to build on. |
||||
# We can also use these variables as a way to inject environment-specific |
||||
# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a |
||||
# reasonable default. |
||||
EXTRA_ENV_COMPILE_ARGS = os.environ.get("GRPC_PYTHON_CFLAGS", None) |
||||
EXTRA_ENV_LINK_ARGS = os.environ.get("GRPC_PYTHON_LDFLAGS", None) |
||||
if EXTRA_ENV_COMPILE_ARGS is None: |
||||
EXTRA_ENV_COMPILE_ARGS = "-std=c++14" |
||||
if "win32" in sys.platform: |
||||
# We need to statically link the C++ Runtime, only the C runtime is |
||||
# available dynamically |
||||
EXTRA_ENV_COMPILE_ARGS += " /MT" |
||||
elif "linux" in sys.platform or "darwin" in sys.platform: |
||||
EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti -fvisibility=hidden" |
||||
|
||||
if EXTRA_ENV_LINK_ARGS is None: |
||||
EXTRA_ENV_LINK_ARGS = "" |
||||
if "linux" in sys.platform or "darwin" in sys.platform: |
||||
EXTRA_ENV_LINK_ARGS += " -lpthread" |
||||
if check_linker_need_libatomic(): |
||||
EXTRA_ENV_LINK_ARGS += " -latomic" |
||||
|
||||
# This enables the standard link-time optimizer, which help us prevent some undefined symbol errors by |
||||
# remove some unused symbols from .so file. |
||||
# Note that it does not work for MSCV on windows. |
||||
if "win32" not in sys.platform: |
||||
EXTRA_ENV_COMPILE_ARGS += " -flto" |
||||
|
||||
EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) |
||||
EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) |
||||
|
||||
if BUILD_WITH_STATIC_LIBSTDCXX: |
||||
EXTRA_LINK_ARGS.append("-static-libstdc++") |
||||
|
||||
CC_FILES = [ |
||||
os.path.normpath(cc_file) for cc_file in observability_lib_deps.CC_FILES |
||||
] |
||||
CC_INCLUDES = [ |
||||
os.path.normpath(include_dir) |
||||
for include_dir in observability_lib_deps.CC_INCLUDES |
||||
] |
||||
|
||||
DEFINE_MACROS = (("_WIN32_WINNT", 0x600),) |
||||
|
||||
if "win32" in sys.platform: |
||||
DEFINE_MACROS += ( |
||||
("WIN32_LEAN_AND_MEAN", 1), |
||||
("CARES_STATICLIB", 1), |
||||
("GRPC_ARES", 0), |
||||
("NTDDI_VERSION", 0x06000000), |
||||
# avoid https://github.com/abseil/abseil-cpp/issues/1425 |
||||
("NOMINMAX", 1), |
||||
) |
||||
if "64bit" in platform.architecture()[0]: |
||||
DEFINE_MACROS += (("MS_WIN64", 1),) |
||||
else: |
||||
# For some reason, this is needed to get access to inet_pton/inet_ntop |
||||
# on msvc, but only for 32 bits |
||||
DEFINE_MACROS += (("NTDDI_VERSION", 0x06000000),) |
||||
elif "linux" in sys.platform or "darwin" in sys.platform: |
||||
DEFINE_MACROS += (("HAVE_PTHREAD", 1),) |
||||
|
||||
# Fix for Cython build issue in aarch64. |
||||
# It's required to define this macro before include <inttypes.h>. |
||||
# <inttypes.h> was included in core/lib/channel/call_tracer.h. |
||||
# This macro should already be defined in grpc/grpc.h through port_platform.h, |
||||
# but we're still having issue in aarch64, so we manually define the macro here. |
||||
# TODO(xuanwn): Figure out what's going on in the aarch64 build so we can support |
||||
# gcc + Bazel. |
||||
DEFINE_MACROS += (("__STDC_FORMAT_MACROS", None),) |
||||
|
||||
|
||||
# Use `-fvisibility=hidden` will hide cython init symbol, we need that symbol exported |
||||
# in order to import cython module. |
||||
if "linux" in sys.platform or "darwin" in sys.platform: |
||||
pymodinit = 'extern "C" __attribute__((visibility ("default"))) PyObject*' |
||||
DEFINE_MACROS += (("PyMODINIT_FUNC", pymodinit),) |
||||
|
||||
# By default, Python3 distutils enforces compatibility of |
||||
# c plugins (.so files) with the OSX version Python was built with. |
||||
# We need OSX 10.10, the oldest which supports C++ thread_local. |
||||
if "darwin" in sys.platform: |
||||
mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") |
||||
if mac_target and ( |
||||
pkg_resources.parse_version(mac_target) |
||||
< pkg_resources.parse_version("10.10.0") |
||||
): |
||||
os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" |
||||
os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( |
||||
r"macosx-[0-9]+\.[0-9]+-(.+)", |
||||
r"macosx-10.10-\1", |
||||
sysconfig.get_platform(), |
||||
) |
||||
|
||||
|
||||
def extension_modules(): |
||||
if BUILD_WITH_CYTHON: |
||||
cython_module_files = [ |
||||
os.path.join("grpc_observability", "_cyobservability.pyx") |
||||
] |
||||
else: |
||||
cython_module_files = [ |
||||
os.path.join("grpc_observability", "_cyobservability.cpp") |
||||
] |
||||
|
||||
plugin_include = [ |
||||
".", |
||||
"grpc_root", |
||||
os.path.join("grpc_root", "include"), |
||||
] + CC_INCLUDES |
||||
|
||||
plugin_sources = CC_FILES |
||||
|
||||
O11Y_CC_PATHS = ( |
||||
os.path.join("grpc_observability", f) for f in O11Y_CC_SRCS |
||||
) |
||||
plugin_sources += O11Y_CC_PATHS |
||||
|
||||
plugin_sources += cython_module_files |
||||
|
||||
plugin_ext = Extension( |
||||
name="grpc_observability._cyobservability", |
||||
sources=plugin_sources, |
||||
include_dirs=plugin_include, |
||||
language="c++", |
||||
define_macros=list(DEFINE_MACROS), |
||||
extra_compile_args=list(EXTRA_COMPILE_ARGS), |
||||
extra_link_args=list(EXTRA_LINK_ARGS), |
||||
) |
||||
extensions = [plugin_ext] |
||||
if BUILD_WITH_CYTHON: |
||||
from Cython import Build |
||||
|
||||
return Build.cythonize( |
||||
extensions, compiler_directives={"language_level": "3"} |
||||
) |
||||
else: |
||||
return extensions |
||||
|
||||
|
||||
PACKAGES = setuptools.find_packages(PYTHON_STEM) |
||||
|
||||
setuptools.setup( |
||||
name="grpcio-observability", |
||||
version=grpc_version.VERSION, |
||||
description="gRPC Python observability package", |
||||
long_description=open(README_PATH, "r").read(), |
||||
author="The gRPC Authors", |
||||
author_email="grpc-io@googlegroups.com", |
||||
url="https://grpc.io", |
||||
project_urls={ |
||||
"Source Code": "https://github.com/grpc/grpc/tree/master/src/python/grpcio_observability", |
||||
"Bug Tracker": "https://github.com/grpc/grpc/issues", |
||||
}, |
||||
license="Apache License 2.0", |
||||
classifiers=CLASSIFIERS, |
||||
ext_modules=extension_modules(), |
||||
packages=list(PACKAGES), |
||||
python_requires=">=3.7", |
||||
install_requires=[ |
||||
"grpcio>={version}".format(version=grpc_version.VERSION), |
||||
"setuptools>=59.6.0", |
||||
], |
||||
cmdclass={ |
||||
"build_ext": BuildExt, |
||||
}, |
||||
) |
||||
@ -0,0 +1,13 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
@ -0,0 +1,65 @@ |
||||
"""Patches the compile() to allow enable parallel compilation of C/C++. |
||||
|
||||
build_ext has lots of C/C++ files and normally them one by one. |
||||
Enabling parallel build helps a lot. |
||||
""" |
||||
|
||||
import os |
||||
|
||||
try: |
||||
BUILD_EXT_COMPILER_JOBS = int( |
||||
os.environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"] |
||||
) |
||||
except KeyError: |
||||
import multiprocessing |
||||
|
||||
BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count() |
||||
except ValueError: |
||||
BUILD_EXT_COMPILER_JOBS = 1 |
||||
|
||||
|
||||
# monkey-patch for parallel compilation |
||||
def _parallel_compile( |
||||
self, |
||||
sources, |
||||
output_dir=None, |
||||
macros=None, |
||||
include_dirs=None, |
||||
debug=0, |
||||
extra_preargs=None, |
||||
extra_postargs=None, |
||||
depends=None, |
||||
): |
||||
# setup the same way as distutils.ccompiler.CCompiler |
||||
# https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 |
||||
macros, objects, extra_postargs, pp_opts, build = self._setup_compile( |
||||
str(output_dir), macros, include_dirs, sources, depends, extra_postargs |
||||
) |
||||
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) |
||||
|
||||
def _compile_single_file(obj): |
||||
try: |
||||
src, ext = build[obj] |
||||
except KeyError: |
||||
return |
||||
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) |
||||
|
||||
# run compilation of individual files in parallel |
||||
import multiprocessing.pool |
||||
|
||||
multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( |
||||
_compile_single_file, objects |
||||
) |
||||
return objects |
||||
|
||||
|
||||
def monkeypatch_compile_maybe(): |
||||
""" |
||||
Monkeypatching is dumb, but the build speed gain is worth it. |
||||
After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib. |
||||
""" |
||||
use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "") |
||||
if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib": |
||||
import distutils.ccompiler # pylint: disable=wrong-import-position |
||||
|
||||
distutils.ccompiler.CCompiler.compile = _parallel_compile |
||||
@ -0,0 +1,23 @@ |
||||
%YAML 1.2 |
||||
--- | |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
# This file has been automatically generated from a template file. |
||||
# Please make modifications to |
||||
# `$REPO_ROOT/templates/src/python/grpcio/_parallel_compile_patch.py.template` |
||||
# instead. This file can be regenerated from the template by running |
||||
# `tools/buildgen/generate_projects.sh`. |
||||
|
||||
<%include file="../_parallel_compile_patch.py.include" /> |
||||
@ -0,0 +1,19 @@ |
||||
%YAML 1.2 |
||||
--- | |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_observability/grpc_version.py.template`!!! |
||||
|
||||
VERSION = '${settings.python_version.pep440()}' |
||||
@ -0,0 +1,23 @@ |
||||
%YAML 1.2 |
||||
--- | |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
# This file has been automatically generated from a template file. |
||||
# Please make modifications to |
||||
# `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py.template` |
||||
# instead. This file can be regenerated from the template by running |
||||
# `tools/buildgen/generate_projects.sh`. |
||||
|
||||
<%include file="../../../../src/python/_parallel_compile_patch.py.include" /> |
||||
Loading…
Reference in new issue