[memory] Trim down 'per-cpu' sharding a little (#33203)

Most of these data structures need to scale a bit like per-cpu, but not
entirely. We can have more than one cpu hit the same instance in most
cases, and probably want to cap out before the hundreds of shards some
platforms have.

<!--

If you know who should review your pull request, please assign it to
that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the
appropriate
lang label.

-->

---------

Co-authored-by: ctiller <ctiller@users.noreply.github.com>
pull/33209/head
Craig Tiller 2 years ago committed by GitHub
parent 9d03117fbc
commit b71b3b410c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      CMakeLists.txt
  2. 2
      Makefile
  3. 4
      build_autogenerated.yaml
  4. 1
      config.m4
  5. 1
      config.w32
  6. 1
      gRPC-Core.podspec
  7. 1
      grpc.gemspec
  8. 3
      grpc.gyp
  9. 1
      package.xml
  10. 4
      src/core/BUILD
  11. 2
      src/core/ext/xds/xds_client_stats.h
  12. 2
      src/core/lib/channel/channelz.h
  13. 2
      src/core/lib/debug/event_log.h
  14. 2
      src/core/lib/debug/stats_data.h
  15. 33
      src/core/lib/gprpp/per_cpu.cc
  16. 35
      src/core/lib/gprpp/per_cpu.h
  17. 1
      src/python/grpcio/grpc_core_dependencies.py
  18. 4
      tools/codegen/core/gen_stats_data.py
  19. 1
      tools/doxygen/Doxyfile.c++.internal
  20. 1
      tools/doxygen/Doxyfile.core.internal

4
CMakeLists.txt generated

@ -2098,6 +2098,7 @@ add_library(grpc
src/core/lib/experiments/config.cc
src/core/lib/experiments/experiments.cc
src/core/lib/gprpp/load_file.cc
src/core/lib/gprpp/per_cpu.cc
src/core/lib/gprpp/status_helper.cc
src/core/lib/gprpp/time.cc
src/core/lib/gprpp/time_averaged_stats.cc
@ -2799,6 +2800,7 @@ add_library(grpc_unsecure
src/core/lib/experiments/config.cc
src/core/lib/experiments/experiments.cc
src/core/lib/gprpp/load_file.cc
src/core/lib/gprpp/per_cpu.cc
src/core/lib/gprpp/status_helper.cc
src/core/lib/gprpp/time.cc
src/core/lib/gprpp/time_averaged_stats.cc
@ -4332,6 +4334,7 @@ add_library(grpc_authorization_provider
src/core/lib/experiments/config.cc
src/core/lib/experiments/experiments.cc
src/core/lib/gprpp/load_file.cc
src/core/lib/gprpp/per_cpu.cc
src/core/lib/gprpp/status_helper.cc
src/core/lib/gprpp/time.cc
src/core/lib/gprpp/time_averaged_stats.cc
@ -11566,6 +11569,7 @@ add_executable(frame_test
src/core/lib/experiments/config.cc
src/core/lib/experiments/experiments.cc
src/core/lib/gprpp/load_file.cc
src/core/lib/gprpp/per_cpu.cc
src/core/lib/gprpp/status_helper.cc
src/core/lib/gprpp/time.cc
src/core/lib/gprpp/time_averaged_stats.cc

2
Makefile generated

@ -1478,6 +1478,7 @@ LIBGRPC_SRC = \
src/core/lib/experiments/config.cc \
src/core/lib/experiments/experiments.cc \
src/core/lib/gprpp/load_file.cc \
src/core/lib/gprpp/per_cpu.cc \
src/core/lib/gprpp/status_helper.cc \
src/core/lib/gprpp/time.cc \
src/core/lib/gprpp/time_averaged_stats.cc \
@ -2033,6 +2034,7 @@ LIBGRPC_UNSECURE_SRC = \
src/core/lib/experiments/config.cc \
src/core/lib/experiments/experiments.cc \
src/core/lib/gprpp/load_file.cc \
src/core/lib/gprpp/per_cpu.cc \
src/core/lib/gprpp/status_helper.cc \
src/core/lib/gprpp/time.cc \
src/core/lib/gprpp/time_averaged_stats.cc \

@ -1527,6 +1527,7 @@ libs:
- src/core/lib/experiments/config.cc
- src/core/lib/experiments/experiments.cc
- src/core/lib/gprpp/load_file.cc
- src/core/lib/gprpp/per_cpu.cc
- src/core/lib/gprpp/status_helper.cc
- src/core/lib/gprpp/time.cc
- src/core/lib/gprpp/time_averaged_stats.cc
@ -2514,6 +2515,7 @@ libs:
- src/core/lib/experiments/config.cc
- src/core/lib/experiments/experiments.cc
- src/core/lib/gprpp/load_file.cc
- src/core/lib/gprpp/per_cpu.cc
- src/core/lib/gprpp/status_helper.cc
- src/core/lib/gprpp/time.cc
- src/core/lib/gprpp/time_averaged_stats.cc
@ -3891,6 +3893,7 @@ libs:
- src/core/lib/experiments/config.cc
- src/core/lib/experiments/experiments.cc
- src/core/lib/gprpp/load_file.cc
- src/core/lib/gprpp/per_cpu.cc
- src/core/lib/gprpp/status_helper.cc
- src/core/lib/gprpp/time.cc
- src/core/lib/gprpp/time_averaged_stats.cc
@ -7812,6 +7815,7 @@ targets:
- src/core/lib/experiments/config.cc
- src/core/lib/experiments/experiments.cc
- src/core/lib/gprpp/load_file.cc
- src/core/lib/gprpp/per_cpu.cc
- src/core/lib/gprpp/status_helper.cc
- src/core/lib/gprpp/time.cc
- src/core/lib/gprpp/time_averaged_stats.cc

1
config.m4 generated

@ -594,6 +594,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/lib/gprpp/linux/env.cc \
src/core/lib/gprpp/load_file.cc \
src/core/lib/gprpp/mpscq.cc \
src/core/lib/gprpp/per_cpu.cc \
src/core/lib/gprpp/posix/env.cc \
src/core/lib/gprpp/posix/stat.cc \
src/core/lib/gprpp/posix/thd.cc \

1
config.w32 generated

@ -559,6 +559,7 @@ if (PHP_GRPC != "no") {
"src\\core\\lib\\gprpp\\linux\\env.cc " +
"src\\core\\lib\\gprpp\\load_file.cc " +
"src\\core\\lib\\gprpp\\mpscq.cc " +
"src\\core\\lib\\gprpp\\per_cpu.cc " +
"src\\core\\lib\\gprpp\\posix\\env.cc " +
"src\\core\\lib\\gprpp\\posix\\stat.cc " +
"src\\core\\lib\\gprpp\\posix\\thd.cc " +

1
gRPC-Core.podspec generated

@ -1311,6 +1311,7 @@ Pod::Spec.new do |s|
'src/core/lib/gprpp/orphanable.h',
'src/core/lib/gprpp/overload.h',
'src/core/lib/gprpp/packed_table.h',
'src/core/lib/gprpp/per_cpu.cc',
'src/core/lib/gprpp/per_cpu.h',
'src/core/lib/gprpp/posix/env.cc',
'src/core/lib/gprpp/posix/stat.cc',

1
grpc.gemspec generated

@ -1217,6 +1217,7 @@ Gem::Specification.new do |s|
s.files += %w( src/core/lib/gprpp/orphanable.h )
s.files += %w( src/core/lib/gprpp/overload.h )
s.files += %w( src/core/lib/gprpp/packed_table.h )
s.files += %w( src/core/lib/gprpp/per_cpu.cc )
s.files += %w( src/core/lib/gprpp/per_cpu.h )
s.files += %w( src/core/lib/gprpp/posix/env.cc )
s.files += %w( src/core/lib/gprpp/posix/stat.cc )

3
grpc.gyp generated

@ -781,6 +781,7 @@
'src/core/lib/experiments/config.cc',
'src/core/lib/experiments/experiments.cc',
'src/core/lib/gprpp/load_file.cc',
'src/core/lib/gprpp/per_cpu.cc',
'src/core/lib/gprpp/status_helper.cc',
'src/core/lib/gprpp/time.cc',
'src/core/lib/gprpp/time_averaged_stats.cc',
@ -1275,6 +1276,7 @@
'src/core/lib/experiments/config.cc',
'src/core/lib/experiments/experiments.cc',
'src/core/lib/gprpp/load_file.cc',
'src/core/lib/gprpp/per_cpu.cc',
'src/core/lib/gprpp/status_helper.cc',
'src/core/lib/gprpp/time.cc',
'src/core/lib/gprpp/time_averaged_stats.cc',
@ -1793,6 +1795,7 @@
'src/core/lib/experiments/config.cc',
'src/core/lib/experiments/experiments.cc',
'src/core/lib/gprpp/load_file.cc',
'src/core/lib/gprpp/per_cpu.cc',
'src/core/lib/gprpp/status_helper.cc',
'src/core/lib/gprpp/time.cc',
'src/core/lib/gprpp/time_averaged_stats.cc',

1
package.xml generated

@ -1199,6 +1199,7 @@
<file baseinstalldir="/" name="src/core/lib/gprpp/orphanable.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/gprpp/overload.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/gprpp/packed_table.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/gprpp/per_cpu.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/gprpp/per_cpu.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/gprpp/posix/env.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/gprpp/posix/stat.cc" role="src" />

@ -2395,10 +2395,14 @@ grpc_cc_library(
grpc_cc_library(
name = "per_cpu",
srcs = [
"lib/gprpp/per_cpu.cc",
],
hdrs = [
"lib/gprpp/per_cpu.h",
],
deps = [
"useful",
"//:exec_ctx",
"//:gpr",
],

@ -239,7 +239,7 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> {
absl::string_view cluster_name_;
absl::string_view eds_service_name_;
RefCountedPtr<XdsLocalityName> name_;
PerCpu<Stats> stats_{32};
PerCpu<Stats> stats_{PerCpuOptions().SetMaxShards(32).SetCpusPerShard(4)};
};
} // namespace grpc_core

@ -158,7 +158,7 @@ class PerCpuCallCountingHelper {
std::atomic<int64_t> calls_failed{0};
std::atomic<gpr_cycle_counter> last_call_started_cycle{0};
};
PerCpu<PerCpuData> per_cpu_data_;
PerCpu<PerCpuData> per_cpu_data_{PerCpuOptions().SetCpusPerShard(4)};
};
// Handles channelz bookkeeping for channels

@ -71,7 +71,7 @@ class EventLog {
std::vector<Entry> EndCollection(
absl::Span<const absl::string_view> wanted_events);
PerCpu<Fragment> fragments_;
PerCpu<Fragment> fragments_{PerCpuOptions().SetCpusPerShard(2)};
gpr_cycle_counter collection_begin_;
static std::atomic<EventLog*> g_instance_;
};

@ -294,7 +294,7 @@ class GlobalStatsCollector {
HistogramCollector_16777216_20 http2_send_message_size;
HistogramCollector_65536_26 http2_metadata_size;
};
PerCpu<Data> data_;
PerCpu<Data> data_{PerCpuOptions().SetCpusPerShard(4).SetMaxShards(32)};
};
} // namespace grpc_core

@ -0,0 +1,33 @@
// Copyright 2023 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <grpc/support/port_platform.h>
#include "src/core/lib/gprpp/per_cpu.h"
#include <grpc/support/cpu.h>
#include "src/core/lib/gpr/useful.h"
namespace grpc_core {
size_t PerCpuOptions::Shards() {
return ShardsForCpuCount(gpr_cpu_num_cores());
}
size_t PerCpuOptions::ShardsForCpuCount(size_t cpu_count) {
return Clamp<size_t>(cpu_count / cpus_per_shard_, 1, max_shards_);
}
} // namespace grpc_core

@ -22,18 +22,41 @@
#include <limits>
#include <memory>
#include <grpc/support/cpu.h>
#include "src/core/lib/iomgr/exec_ctx.h"
namespace grpc_core {
class PerCpuOptions {
public:
// Set the number of cpus that colocate on the same shard
PerCpuOptions SetCpusPerShard(size_t cpus_per_shard) {
cpus_per_shard_ = std::max<size_t>(1, cpus_per_shard);
return *this;
}
// Set the maximum number of allowable shards
PerCpuOptions SetMaxShards(size_t max_shards) {
max_shards_ = std::max<size_t>(1, max_shards);
return *this;
}
size_t cpus_per_shard() const { return cpus_per_shard_; }
size_t max_shards() const { return max_shards_; }
size_t Shards();
size_t ShardsForCpuCount(size_t cpu_count);
private:
size_t cpus_per_shard_ = 1;
size_t max_shards_ = std::numeric_limits<size_t>::max();
};
template <typename T>
class PerCpu {
public:
explicit PerCpu(size_t max = std::numeric_limits<size_t>::max())
: cpus_(std::min<size_t>(max, gpr_cpu_num_cores())),
data_{new T[cpus_]} {}
// Options are not defaulted to try and force consideration of what the
// options specify.
explicit PerCpu(PerCpuOptions options) : cpus_(options.Shards()) {}
T& this_cpu() { return data_[ExecCtx::Get()->starting_cpu() % cpus_]; }
@ -44,7 +67,7 @@ class PerCpu {
private:
const size_t cpus_;
std::unique_ptr<T[]> data_;
std::unique_ptr<T[]> data_{new T[cpus_]};
};
} // namespace grpc_core

@ -568,6 +568,7 @@ CORE_SOURCE_FILES = [
'src/core/lib/gprpp/linux/env.cc',
'src/core/lib/gprpp/load_file.cc',
'src/core/lib/gprpp/mpscq.cc',
'src/core/lib/gprpp/per_cpu.cc',
'src/core/lib/gprpp/posix/env.cc',
'src/core/lib/gprpp/posix/stat.cc',
'src/core/lib/gprpp/posix/thd.cc',

@ -369,7 +369,9 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H:
(ctr.max, ctr.buckets, ctr.name),
file=H)
print(" };", file=H)
print(" PerCpu<Data> data_;", file=H)
print(
" PerCpu<Data> data_{PerCpuOptions().SetCpusPerShard(4).SetMaxShards(32)};",
file=H)
print("};", file=H)
print("}", file=H)

@ -2213,6 +2213,7 @@ src/core/lib/gprpp/notification.h \
src/core/lib/gprpp/orphanable.h \
src/core/lib/gprpp/overload.h \
src/core/lib/gprpp/packed_table.h \
src/core/lib/gprpp/per_cpu.cc \
src/core/lib/gprpp/per_cpu.h \
src/core/lib/gprpp/posix/env.cc \
src/core/lib/gprpp/posix/stat.cc \

@ -1993,6 +1993,7 @@ src/core/lib/gprpp/notification.h \
src/core/lib/gprpp/orphanable.h \
src/core/lib/gprpp/overload.h \
src/core/lib/gprpp/packed_table.h \
src/core/lib/gprpp/per_cpu.cc \
src/core/lib/gprpp/per_cpu.h \
src/core/lib/gprpp/posix/env.cc \
src/core/lib/gprpp/posix/stat.cc \

Loading…
Cancel
Save