|
|
@ -7,6 +7,10 @@ |
|
|
|
|
|
|
|
|
|
|
|
#include "upb/mem/arena.h" |
|
|
|
#include "upb/mem/arena.h" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef UPB_TRACING_ENABLED |
|
|
|
|
|
|
|
#include <stdatomic.h> |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
|
|
|
|
#include <stddef.h> |
|
|
|
#include <stddef.h> |
|
|
|
#include <stdint.h> |
|
|
|
#include <stdint.h> |
|
|
|
|
|
|
|
|
|
|
@ -117,6 +121,38 @@ static bool _upb_ArenaInternal_HasInitialBlock(upb_ArenaInternal* ai) { |
|
|
|
return ai->block_alloc & 0x1; |
|
|
|
return ai->block_alloc & 0x1; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef UPB_TRACING_ENABLED |
|
|
|
|
|
|
|
static void (*_init_arena_trace_handler)(const upb_Arena*, size_t size) = NULL; |
|
|
|
|
|
|
|
static void (*_fuse_arena_trace_handler)(const upb_Arena*, |
|
|
|
|
|
|
|
const upb_Arena*) = NULL; |
|
|
|
|
|
|
|
static void (*_free_arena_trace_handler)(const upb_Arena*) = NULL; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void upb_Arena_SetTraceHandler( |
|
|
|
|
|
|
|
void (*initArenaTraceHandler)(const upb_Arena*, size_t size), |
|
|
|
|
|
|
|
void (*fuseArenaTraceHandler)(const upb_Arena*, const upb_Arena*), |
|
|
|
|
|
|
|
void (*freeArenaTraceHandler)(const upb_Arena*)) { |
|
|
|
|
|
|
|
_init_arena_trace_handler = initArenaTraceHandler; |
|
|
|
|
|
|
|
_fuse_arena_trace_handler = fuseArenaTraceHandler; |
|
|
|
|
|
|
|
_free_arena_trace_handler = freeArenaTraceHandler; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void upb_Arena_LogInit(const upb_Arena* arena, size_t size) { |
|
|
|
|
|
|
|
if (_init_arena_trace_handler) { |
|
|
|
|
|
|
|
_init_arena_trace_handler(arena, size); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
void upb_Arena_LogFuse(const upb_Arena* arena1, const upb_Arena* arena2) { |
|
|
|
|
|
|
|
if (_fuse_arena_trace_handler) { |
|
|
|
|
|
|
|
_fuse_arena_trace_handler(arena1, arena2); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
void upb_Arena_LogFree(const upb_Arena* arena) { |
|
|
|
|
|
|
|
if (_free_arena_trace_handler) { |
|
|
|
|
|
|
|
_free_arena_trace_handler(arena); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
#endif // UPB_TRACING_ENABLED
|
|
|
|
|
|
|
|
|
|
|
|
static upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) { |
|
|
|
static upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) { |
|
|
|
upb_ArenaInternal* ai = upb_Arena_Internal(a); |
|
|
|
upb_ArenaInternal* ai = upb_Arena_Internal(a); |
|
|
|
uintptr_t poc = upb_Atomic_Load(&ai->parent_or_count, memory_order_acquire); |
|
|
|
uintptr_t poc = upb_Atomic_Load(&ai->parent_or_count, memory_order_acquire); |
|
|
@ -280,7 +316,13 @@ upb_Arena* upb_Arena_Init(void* mem, size_t n, upb_alloc* alloc) { |
|
|
|
n = UPB_ALIGN_DOWN(n, UPB_ALIGN_OF(upb_ArenaState)); |
|
|
|
n = UPB_ALIGN_DOWN(n, UPB_ALIGN_OF(upb_ArenaState)); |
|
|
|
|
|
|
|
|
|
|
|
if (UPB_UNLIKELY(n < sizeof(upb_ArenaState))) { |
|
|
|
if (UPB_UNLIKELY(n < sizeof(upb_ArenaState))) { |
|
|
|
|
|
|
|
#ifdef UPB_TRACING_ENABLED |
|
|
|
|
|
|
|
upb_Arena* ret = _upb_Arena_InitSlow(alloc); |
|
|
|
|
|
|
|
upb_Arena_LogInit(ret, n); |
|
|
|
|
|
|
|
return ret; |
|
|
|
|
|
|
|
#else |
|
|
|
return _upb_Arena_InitSlow(alloc); |
|
|
|
return _upb_Arena_InitSlow(alloc); |
|
|
|
|
|
|
|
#endif |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
a = UPB_PTR_AT(mem, n - sizeof(upb_ArenaState), upb_ArenaState); |
|
|
|
a = UPB_PTR_AT(mem, n - sizeof(upb_ArenaState), upb_ArenaState); |
|
|
@ -293,13 +335,14 @@ upb_Arena* upb_Arena_Init(void* mem, size_t n, upb_alloc* alloc) { |
|
|
|
a->body.block_alloc = _upb_Arena_MakeBlockAlloc(alloc, 1); |
|
|
|
a->body.block_alloc = _upb_Arena_MakeBlockAlloc(alloc, 1); |
|
|
|
a->head.UPB_PRIVATE(ptr) = mem; |
|
|
|
a->head.UPB_PRIVATE(ptr) = mem; |
|
|
|
a->head.UPB_PRIVATE(end) = UPB_PTR_AT(mem, n - sizeof(upb_ArenaState), char); |
|
|
|
a->head.UPB_PRIVATE(end) = UPB_PTR_AT(mem, n - sizeof(upb_ArenaState), char); |
|
|
|
|
|
|
|
#ifdef UPB_TRACING_ENABLED |
|
|
|
|
|
|
|
upb_Arena_LogInit(&a->head, n); |
|
|
|
|
|
|
|
#endif |
|
|
|
return &a->head; |
|
|
|
return &a->head; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
static void _upb_Arena_DoFree(upb_ArenaInternal* ai) { |
|
|
|
static void _upb_Arena_DoFree(upb_ArenaInternal* ai) { |
|
|
|
UPB_ASSERT(_upb_Arena_RefCountFromTagged(ai->parent_or_count) == 1); |
|
|
|
UPB_ASSERT(_upb_Arena_RefCountFromTagged(ai->parent_or_count) == 1); |
|
|
|
|
|
|
|
|
|
|
|
while (ai != NULL) { |
|
|
|
while (ai != NULL) { |
|
|
|
// Load first since arena itself is likely from one of its blocks.
|
|
|
|
// Load first since arena itself is likely from one of its blocks.
|
|
|
|
upb_ArenaInternal* next_arena = |
|
|
|
upb_ArenaInternal* next_arena = |
|
|
@ -330,6 +373,9 @@ retry: |
|
|
|
// expensive then direct loads. As an optimization, we only do RMW ops
|
|
|
|
// expensive then direct loads. As an optimization, we only do RMW ops
|
|
|
|
// when we need to update things for other threads to see.
|
|
|
|
// when we need to update things for other threads to see.
|
|
|
|
if (poc == _upb_Arena_TaggedFromRefcount(1)) { |
|
|
|
if (poc == _upb_Arena_TaggedFromRefcount(1)) { |
|
|
|
|
|
|
|
#ifdef UPB_TRACING_ENABLED |
|
|
|
|
|
|
|
upb_Arena_LogFree(a); |
|
|
|
|
|
|
|
#endif |
|
|
|
_upb_Arena_DoFree(ai); |
|
|
|
_upb_Arena_DoFree(ai); |
|
|
|
return; |
|
|
|
return; |
|
|
|
} |
|
|
|
} |
|
|
@ -449,6 +495,10 @@ static bool _upb_Arena_FixupRefs(upb_ArenaInternal* new_root, |
|
|
|
bool upb_Arena_Fuse(upb_Arena* a1, upb_Arena* a2) { |
|
|
|
bool upb_Arena_Fuse(upb_Arena* a1, upb_Arena* a2) { |
|
|
|
if (a1 == a2) return true; // trivial fuse
|
|
|
|
if (a1 == a2) return true; // trivial fuse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef UPB_TRACING_ENABLED |
|
|
|
|
|
|
|
upb_Arena_LogFuse(a1, a2); |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
|
|
|
|
upb_ArenaInternal* ai1 = upb_Arena_Internal(a1); |
|
|
|
upb_ArenaInternal* ai1 = upb_Arena_Internal(a1); |
|
|
|
upb_ArenaInternal* ai2 = upb_Arena_Internal(a2); |
|
|
|
upb_ArenaInternal* ai2 = upb_Arena_Internal(a2); |
|
|
|
|
|
|
|
|
|
|
|