@ -40,76 +40,12 @@
# include "src/core/lib/resource_quota/memory_quota.h"
# include "src/core/util/alloc.h"
# define GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
// #define GRPC_ARENA_TRACE_POOLED_ALLOCATIONS
namespace grpc_core {
namespace arena_detail {
# ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
struct PoolAndSize {
size_t alloc_size ;
size_t pool_index ;
} ;
template < typename Void , size_t kIndex , size_t kObjectSize ,
size_t . . . kBucketSize >
struct PoolIndexForSize ;
template < size_t kObjectSize , size_t kIndex , size_t kSmallestRemainingBucket ,
size_t . . . kBucketSizes >
struct PoolIndexForSize <
absl : : enable_if_t < kObjectSize < = kSmallestRemainingBucket > , kIndex ,
kObjectSize , kSmallestRemainingBucket , kBucketSizes . . . > {
static constexpr size_t kPool = kIndex ;
static constexpr size_t kSize = kSmallestRemainingBucket ;
} ;
template < size_t kObjectSize , size_t kIndex , size_t kSmallestRemainingBucket ,
size_t . . . kBucketSizes >
struct PoolIndexForSize <
absl : : enable_if_t < ( kObjectSize > kSmallestRemainingBucket ) > , kIndex ,
kObjectSize , kSmallestRemainingBucket , kBucketSizes . . . >
: public PoolIndexForSize < void , kIndex + 1 , kObjectSize , kBucketSizes . . . > {
} ;
template < size_t kObjectSize , size_t . . . kBucketSizes >
constexpr size_t PoolFromObjectSize (
absl : : integer_sequence < size_t , kBucketSizes . . . > ) {
return PoolIndexForSize < void , 0 , kObjectSize , kBucketSizes . . . > : : kPool ;
}
template < size_t kObjectSize , size_t . . . kBucketSizes >
constexpr size_t AllocationSizeFromObjectSize (
absl : : integer_sequence < size_t , kBucketSizes . . . > ) {
return PoolIndexForSize < void , 0 , kObjectSize , kBucketSizes . . . > : : kSize ;
}
template < size_t kIndex , size_t . . . kBucketSizes >
struct ChoosePoolForAllocationSizeImpl ;
template < size_t kIndex , size_t kBucketSize , size_t . . . kBucketSizes >
struct ChoosePoolForAllocationSizeImpl < kIndex , kBucketSize , kBucketSizes . . . > {
static PoolAndSize Fn ( size_t n ) {
if ( n < = kBucketSize ) return { kBucketSize , kIndex } ;
return ChoosePoolForAllocationSizeImpl < kIndex + 1 , kBucketSizes . . . > : : Fn ( n ) ;
}
} ;
class Arena ;
template < size_t kIndex >
struct ChoosePoolForAllocationSizeImpl < kIndex > {
static PoolAndSize Fn ( size_t n ) {
return PoolAndSize { n , std : : numeric_limits < size_t > : : max ( ) } ;
}
} ;
namespace arena_detail {
template < size_t . . . kBucketSizes >
PoolAndSize ChoosePoolForAllocationSize (
size_t n , absl : : integer_sequence < size_t , kBucketSizes . . . > ) {
return ChoosePoolForAllocationSizeImpl < 0 , kBucketSizes . . . > : : Fn ( n ) ;
}
# else
template < typename T , typename A , typename B >
struct IfArray {
using Result = A ;
@ -119,30 +55,36 @@ template <typename T, typename A, typename B>
struct IfArray < T [ ] , A , B > {
using Result = B ;
} ;
# endif
struct UnrefDestroy {
void operator ( ) ( const Arena * arena ) const ;
} ;
} // namespace arena_detail
class Arena {
# ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
// Selected pool sizes.
// How to tune: see tools/codegen/core/optimize_arena_pool_sizes.py
using PoolSizes = absl : : integer_sequence < size_t , 80 , 304 , 528 , 1024 > ;
struct FreePoolNode {
FreePoolNode * next ;
} ;
# endif
class ArenaFactory : public RefCounted < ArenaFactory > {
public :
virtual RefCountedPtr < Arena > MakeArena ( ) = 0 ;
virtual void FinalizeArena ( Arena * arena ) = 0 ;
MemoryAllocator & allocator ( ) { return allocator_ ; }
protected :
explicit ArenaFactory ( MemoryAllocator allocator )
: allocator_ ( std : : move ( allocator ) ) { }
private :
MemoryAllocator allocator_ ;
} ;
RefCountedPtr < ArenaFactory > SimpleArenaAllocator ( size_t initial_size = 1024 ) ;
class Arena final : public RefCounted < Arena , NonPolymorphicRefCount ,
arena_detail : : UnrefDestroy > {
public :
// Create an arena, with \a initial_size bytes in the first allocated buffer.
static Arena * Create ( size_t initial_size , MemoryAllocator * memory_allocator ) ;
// Create an arena, with \a initial_size bytes in the first allocated buffer,
// and return both a void pointer to the returned arena and a void* with the
// first allocation.
static std : : pair < Arena * , void * > CreateWithAlloc (
size_t initial_size , size_t alloc_size ,
MemoryAllocator * memory_allocator ) ;
static RefCountedPtr < Arena > Create ( size_t initial_size ,
RefCountedPtr < ArenaFactory > arena_factory ) ;
// Destroy all `ManagedNew` allocated objects.
// Allows safe destruction of these objects even if they need context held by
@ -151,9 +93,6 @@ class Arena {
// TODO(ctiller): eliminate ManagedNew.
void DestroyManagedNewObjects ( ) ;
// Destroy an arena.
void Destroy ( ) ;
// Return the total amount of memory allocated by this arena.
size_t TotalUsedBytes ( ) const {
return total_used_ . load ( std : : memory_order_relaxed ) ;
@ -194,95 +133,6 @@ class Arena {
return & p - > t ;
}
# ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
class PooledDeleter {
public :
explicit PooledDeleter ( std : : atomic < FreePoolNode * > * free_list )
: free_list_ ( free_list ) { }
PooledDeleter ( ) = default ;
template < typename T >
void operator ( ) ( T * p ) {
// TODO(ctiller): promise based filter hijacks ownership of some pointers
// to make them appear as PoolPtr without really transferring ownership,
// by setting the arena to nullptr.
// This is a transitional hack and should be removed once promise based
// filter is removed.
if ( free_list_ ! = nullptr ) {
p - > ~ T ( ) ;
FreePooled ( p , free_list_ ) ;
}
}
bool has_freelist ( ) const { return free_list_ ! = nullptr ; }
private :
std : : atomic < FreePoolNode * > * free_list_ ;
} ;
template < typename T >
using PoolPtr = std : : unique_ptr < T , PooledDeleter > ;
// Make a unique_ptr to T that is allocated from the arena.
// When the pointer is released, the memory may be reused for other
// MakePooled(.*) calls.
// CAUTION: The amount of memory allocated is rounded up to the nearest
// value in Arena::PoolSizes, and so this may pessimize total
// arena size.
template < typename T , typename . . . Args >
PoolPtr < T > MakePooled ( Args & & . . . args ) {
auto * free_list =
& pools_ [ arena_detail : : PoolFromObjectSize < sizeof ( T ) > ( PoolSizes ( ) ) ] ;
return PoolPtr < T > (
new ( AllocPooled (
sizeof ( T ) ,
arena_detail : : AllocationSizeFromObjectSize < sizeof ( T ) > ( PoolSizes ( ) ) ,
free_list ) ) T ( std : : forward < Args > ( args ) . . . ) ,
PooledDeleter ( free_list ) ) ;
}
// Make a unique_ptr to an array of T that is allocated from the arena.
// When the pointer is released, the memory may be reused for other
// MakePooled(.*) calls.
// One can use MakePooledArray<char> to allocate a buffer of bytes.
// CAUTION: The amount of memory allocated is rounded up to the nearest
// value in Arena::PoolSizes, and so this may pessimize total
// arena size.
template < typename T >
PoolPtr < T [ ] > MakePooledArray ( size_t n ) {
auto where =
arena_detail : : ChoosePoolForAllocationSize ( n * sizeof ( T ) , PoolSizes ( ) ) ;
if ( where . pool_index = = std : : numeric_limits < size_t > : : max ( ) ) {
return PoolPtr < T [ ] > ( new ( Alloc ( where . alloc_size ) ) T [ n ] ,
PooledDeleter ( nullptr ) ) ;
} else {
return PoolPtr < T [ ] > ( new ( AllocPooled ( where . alloc_size , where . alloc_size ,
& pools_ [ where . pool_index ] ) ) T [ n ] ,
PooledDeleter ( & pools_ [ where . pool_index ] ) ) ;
}
}
// Like MakePooled, but with manual memory management.
// The caller is responsible for calling DeletePooled() on the returned
// pointer, and expected to call it with the same type T as was passed to this
// function (else the free list returned to the arena will be corrupted).
template < typename T , typename . . . Args >
T * NewPooled ( Args & & . . . args ) {
auto * free_list =
& pools_ [ arena_detail : : PoolFromObjectSize < sizeof ( T ) > ( PoolSizes ( ) ) ] ;
return new ( AllocPooled (
sizeof ( T ) ,
arena_detail : : AllocationSizeFromObjectSize < sizeof ( T ) > ( PoolSizes ( ) ) ,
free_list ) ) T ( std : : forward < Args > ( args ) . . . ) ;
}
template < typename T >
void DeletePooled ( T * p ) {
auto * free_list =
& pools_ [ arena_detail : : PoolFromObjectSize < sizeof ( T ) > ( PoolSizes ( ) ) ] ;
p - > ~ T ( ) ;
FreePooled ( p , free_list ) ;
}
# else
class PooledDeleter {
public :
PooledDeleter ( ) = default ;
@ -364,9 +214,10 @@ class Arena {
void DeletePooled ( T * p ) {
delete p ;
}
# endif
private :
friend struct arena_detail : : UnrefDestroy ;
struct Zone {
Zone * prev ;
} ;
@ -397,34 +248,12 @@ class Arena {
// where we wish to create an arena and then perform an immediate
// allocation.
explicit Arena ( size_t initial_size , size_t initial_alloc ,
MemoryAllocator * memory_allocator )
: total_used_ ( GPR_ROUND_UP_TO_ALIGNMENT_SIZE ( initial_alloc ) ) ,
initial_zone_size_ ( initial_size ) ,
memory_allocator_ ( memory_allocator ) { }
RefCountedPtr < ArenaFactory > arena_factory ) ;
~ Arena ( ) ;
void * AllocZone ( size_t size ) ;
# ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
void * AllocPooled ( size_t obj_size , size_t alloc_size ,
std : : atomic < FreePoolNode * > * head ) ;
static void FreePooled ( void * p , std : : atomic < FreePoolNode * > * head ) ;
# endif
void TracePoolAlloc ( size_t size , void * ptr ) {
( void ) size ;
( void ) ptr ;
# ifdef GRPC_ARENA_TRACE_POOLED_ALLOCATIONS
gpr_log ( GPR_ERROR , " ARENA %p ALLOC % " PRIdPTR " @ %p " , this , size , ptr ) ;
# endif
}
static void TracePoolFree ( void * ptr ) {
( void ) ptr ;
# ifdef GRPC_ARENA_TRACE_POOLED_ALLOCATIONS
gpr_log ( GPR_ERROR , " FREE %p " , ptr ) ;
# endif
}
void Destroy ( ) const ;
// Keep track of the total used size. We use this in our call sizing
// hysteresis.
@ -438,27 +267,19 @@ class Arena {
// last zone; the zone list is reverse-walked during arena destruction only.
std : : atomic < Zone * > last_zone_ { nullptr } ;
std : : atomic < ManagedNewObject * > managed_new_head_ { nullptr } ;
# ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
std : : atomic < FreePoolNode * > pools_ [ PoolSizes : : size ( ) ] { } ;
# endif
// The backing memory quota
MemoryAllocator * const memory_allocator_ ;
RefCountedPtr < ArenaFactory > arena_factory_ ;
} ;
// Smart pointer for arenas when the final size is not required.
struct ScopedArenaDeleter {
void operator ( ) ( Arena * arena ) { arena - > Destroy ( ) ; }
} ;
using ScopedArenaPtr = std : : unique_ptr < Arena , ScopedArenaDeleter > ;
inline ScopedArenaPtr MakeScopedArena ( size_t initial_size ,
MemoryAllocator * memory_allocator ) {
return ScopedArenaPtr ( Arena : : Create ( initial_size , memory_allocator ) ) ;
}
// Arenas form a context for activities
template < >
struct ContextType < Arena > { } ;
namespace arena_detail {
inline void UnrefDestroy : : operator ( ) ( const Arena * arena ) const {
arena - > Destroy ( ) ;
}
} // namespace arena_detail
} // namespace grpc_core
# endif // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_ARENA_H
Craig Tiller
6 months ago
committed by
Copybara-Service