@ -23,6 +23,8 @@
# include <algorithm>
# include <cstddef>
# include <cstdint>
# include <cstring>
# include <iterator>
# include <limits>
# include <memory>
@ -113,6 +115,120 @@ struct HeapRep {
} ;
} ;
// We use small object optimization (SOO) to store elements inline when possible
// for small repeated fields. We do so in order to avoid memory indirections.
// SOO data is stored in the same space as the size/capacity ints.
enum { kSooCapacityBytes = 2 * sizeof ( int ) } ;
// Arena/elements pointers are aligned to at least kSooPtrAlignment bytes so we
// can use the lower bits to encode whether we're in SOO mode and if so, the
// SOO size. NOTE: we also tried using all kSooPtrMask bits to encode SOO size
// and use all ones as a sentinel value for non-SOO mode, but that was slower in
// benchmarks/loadtests.
enum { kSooPtrAlignment = sizeof ( void * ) } ;
// The mask for the size bits in SOO mode, and also a sentinel value indicating
// that the field is not in SOO mode.
enum { kSooPtrMask = ~ ( kSooPtrAlignment - 1 ) } ;
// This bit is 0 when in SOO mode and 1 when in non-SOO mode.
enum { kNotSooBit = kSooPtrAlignment > > 1 } ;
// These bits are used to encode the size when in SOO mode (sizes are 0-3).
enum { kSooSizeMask = kNotSooBit - 1 } ;
// The number of elements that can be stored in the SOO rep. When
// kSooPtrAlignment == 8, this is 1 for int64_t, 2 for int32_t, 3 for bool, and
// 0 for absl::Cord.
constexpr int SooCapacityElements ( size_t element_size ) {
return std : : min < int > ( kSooCapacityBytes / element_size , kSooSizeMask ) ;
}
struct LongSooRep {
// Returns char* rather than void* so callers can do pointer arithmetic.
char * elements ( ) const {
auto ret = reinterpret_cast < char * > ( elements_int & kSooPtrMask ) ;
ABSL_DCHECK_NE ( ret , nullptr ) ;
return ret ;
}
uintptr_t elements_int ;
int size ;
int capacity ;
} ;
struct ShortSooRep {
constexpr ShortSooRep ( ) = default ;
explicit ShortSooRep ( Arena * arena )
: arena_and_size ( reinterpret_cast < uintptr_t > ( arena ) ) {
ABSL_DCHECK_EQ ( size ( ) , 0 ) ;
}
int size ( ) const { return arena_and_size & kSooSizeMask ; }
bool is_soo ( ) const { return ( arena_and_size & kNotSooBit ) = = 0 ; }
uintptr_t arena_and_size = 0 ;
union {
char data [ kSooCapacityBytes ] ;
// NOTE: in some language versions, we can't have a constexpr constructor
// if we don't initialize all fields, but `data` doesn't need to be
// initialized so initialize an empty dummy variable instead.
std : : true_type dummy = { } ;
} ;
} ;
struct SooRep {
constexpr SooRep ( ) : short_rep ( ) { }
explicit SooRep ( Arena * arena ) : short_rep ( arena ) { }
bool is_soo ( ) const {
static_assert ( sizeof ( LongSooRep ) = = sizeof ( ShortSooRep ) , " " ) ;
static_assert ( offsetof ( SooRep , long_rep ) = = offsetof ( SooRep , short_rep ) ,
" " ) ;
static_assert ( offsetof ( LongSooRep , elements_int ) = =
offsetof ( ShortSooRep , arena_and_size ) ,
" " ) ;
return short_rep . is_soo ( ) ;
}
Arena * soo_arena ( ) const {
ABSL_DCHECK ( is_soo ( ) ) ;
return reinterpret_cast < Arena * > ( short_rep . arena_and_size & kSooPtrMask ) ;
}
int size ( bool is_soo ) const {
ABSL_DCHECK_EQ ( is_soo , this - > is_soo ( ) ) ;
# if !defined(__clang__) && defined(__GNUC__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
# endif
return is_soo ? short_rep . size ( ) : long_rep . size ;
# if !defined(__clang__) && defined(__GNUC__)
# pragma GCC diagnostic pop
# endif
}
void set_size ( bool is_soo , int size ) {
ABSL_DCHECK_EQ ( is_soo , this - > is_soo ( ) ) ;
if ( is_soo ) {
ABSL_DCHECK_LE ( size , kSooSizeMask ) ;
short_rep . arena_and_size & = kSooPtrMask ;
short_rep . arena_and_size | = size ;
} else {
long_rep . size = size ;
}
}
// Initializes the SooRep in non-SOO mode with the given capacity and heap
// allocation.
void set_non_soo ( bool was_soo , int capacity , void * elements ) {
ABSL_DCHECK_EQ ( was_soo , is_soo ( ) ) ;
ABSL_DCHECK_NE ( elements , nullptr ) ;
ABSL_DCHECK_EQ ( reinterpret_cast < uintptr_t > ( elements ) % kSooPtrAlignment ,
uintptr_t { 0 } ) ;
if ( was_soo ) long_rep . size = short_rep . size ( ) ;
long_rep . capacity = capacity ;
long_rep . elements_int = reinterpret_cast < uintptr_t > ( elements ) | kNotSooBit ;
}
union {
LongSooRep long_rep ;
ShortSooRep short_rep ;
} ;
} ;
} // namespace internal
// RepeatedField is used to represent repeated fields of a primitive type (in
@ -318,10 +434,7 @@ class RepeatedField final
// Gets the Arena on which this RepeatedField stores its elements.
// Note: this can be inaccurate for split default fields so we make this
// function non-const.
inline Arena * GetArena ( ) {
return Capacity ( ) = = 0 ? static_cast < Arena * > ( arena_or_elements_ )
: heap_rep ( ) - > arena ;
}
inline Arena * GetArena ( ) { return GetArena ( is_soo ( ) ) ; }
// For internal use only.
//
@ -339,15 +452,35 @@ class RepeatedField final
friend class Arena ;
static constexpr int kSooCapacityElements =
internal : : SooCapacityElements ( sizeof ( Element ) ) ;
static constexpr int kInitialSize = 0 ;
static PROTOBUF_CONSTEXPR const size_t kHeapRepHeaderSize = sizeof ( HeapRep ) ;
RepeatedField ( Arena * arena , const RepeatedField & rhs ) ;
RepeatedField ( Arena * arena , RepeatedField & & rhs ) ;
inline Arena * GetArena ( bool is_soo ) const {
return is_soo ? soo_rep_ . soo_arena ( ) : heap_rep ( ) - > arena ;
}
void set_size ( int s ) { size_ = s ; }
void set_capacity ( int c ) { capacity_ = c ; }
bool is_soo ( ) const { return soo_rep_ . is_soo ( ) ; }
int size ( bool is_soo ) const { return soo_rep_ . size ( is_soo ) ; }
int Capacity ( bool is_soo ) const {
# if !defined(__clang__) && defined(__GNUC__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
# endif
return is_soo ? kSooCapacityElements : soo_rep_ . long_rep . capacity ;
# if !defined(__clang__) && defined(__GNUC__)
# pragma GCC diagnostic pop
# endif
}
void set_size ( bool is_soo , int size ) {
ABSL_DCHECK_LE ( size , Capacity ( is_soo ) ) ;
soo_rep_ . set_size ( is_soo , size ) ;
}
// Swaps entire contents with "other". Should be called only if the caller can
// guarantee that both repeated fields are on the same arena or are on the
@ -392,63 +525,83 @@ class RepeatedField final
// the old container from `old_size` to `Capacity()` (unpoison memory)
// directly before it is being released, and annotate the new container from
// `Capacity()` to `old_size` (poison unused memory).
void Grow ( int old_size , int new_size ) ;
void GrowNoAnnotate ( int old_size , int new_size ) ;
void Grow ( bool was_soo , int old_size , int new_size ) ;
void GrowNoAnnotate ( bool was_soo , int old_size , int new_size ) ;
// Annotates a change in size of this instance. This function should be called
// with (capacity, old_size) after new memory has been allocated and
// filled from previous memory), and called with (old_size, capacity)
// right before (previously annotated) memory is released.
// with (capacity, old_size) after new memory has been allocated and filled
// from previous memory, and UnpoisonBuffer() should be called right before
// (previously annotated) memory is released.
void AnnotateSize ( int old_size , int new_size ) const {
if ( old_size ! = new_size ) {
ABSL_ANNOTATE_CONTIGUOUS_CONTAINER (
unsafe_elements ( ) , unsafe_elements ( ) + Capacity ( ) ,
unsafe_elements ( ) + old_size , unsafe_elements ( ) + new_size ) ;
ABSL_ATTRIBUTE_UNUSED const bool is_soo = this - > is_soo ( ) ;
ABSL_ATTRIBUTE_UNUSED const Element * elem = unsafe_elements ( is_soo ) ;
ABSL_ANNOTATE_CONTIGUOUS_CONTAINER ( elem , elem + Capacity ( is_soo ) ,
elem + old_size , elem + new_size ) ;
if ( new_size < old_size ) {
ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED (
unsafe_elements ( ) + new_size ,
( old_size - new_size ) * sizeof ( Element ) ) ;
elem + new_size , ( old_size - new_size ) * sizeof ( Element ) ) ;
}
}
}
// Replaces size with new_size and returns the previous value of size. This
// function is intended to be the only place where size is modified, with the
// exception of `AddInputIterator()` where the size of added items is not
// known in advance.
inline int ExchangeCurrentSize ( int new_size ) {
const int prev_size = size ( ) ;
// Unpoisons the memory buffer.
void UnpoisonBuffer ( ) const {
AnnotateSize ( size ( ) , Capacity ( ) ) ;
if ( is_soo ( ) ) {
// We need to manually unpoison the SOO buffer because in reflection for
// split repeated fields, we poison the whole SOO buffer even when we
// don't actually use the whole SOO buffer (e.g. for RepeatedField<bool>).
PROTOBUF_UNPOISON_MEMORY_REGION ( soo_rep_ . short_rep . data ,
sizeof ( soo_rep_ . short_rep . data ) ) ;
}
}
// Replaces size with new_size and returns the previous value of
// size. This function is intended to be the only place where
// size is modified, with the exception of `AddInputIterator()`
// where the size of added items is not known in advance.
inline int ExchangeCurrentSize ( bool is_soo , int new_size ) {
const int prev_size = size ( is_soo ) ;
AnnotateSize ( prev_size , new_size ) ;
set_size ( new_size ) ;
set_size ( is_soo , new_size ) ;
return prev_size ;
}
// Returns a pointer to elements array.
// pre-condition: Capacity() > 0.
Element * elements ( ) const {
ABSL_DCHECK_GT ( Capacity ( ) , 0 ) ;
// Because of above pre-condition this cast is safe.
return unsafe_elements ( ) ;
Element * elements ( bool is_soo ) {
ABSL_DCHECK_GT ( Capacity ( is_soo ) , 0 ) ;
return unsafe_elements ( is_soo ) ;
}
const Element * elements ( bool is_soo ) const {
return const_cast < RepeatedField * > ( this ) - > elements ( is_soo ) ;
}
// Returns a pointer to elements array if it exists; otherwise either null or
// an invalid pointer is returned. This only happens for empty repeated
// fields, where you can't dereference this pointer anyway (it's empty).
Element * unsafe_elements ( ) const {
return static_cast < Element * > ( arena_or_elements_ ) ;
// Returns a pointer to elements array if it exists; otherwise an invalid
// pointer is returned. This only happens for empty repeated fields, where you
// can't dereference this pointer anyway (it's empty).
Element * unsafe_elements ( bool is_soo ) {
return is_soo ? reinterpret_cast < Element * > ( soo_rep_ . short_rep . data )
: reinterpret_cast < Element * > ( soo_rep_ . long_rep . elements ( ) ) ;
}
const Element * unsafe_elements ( bool is_soo ) const {
return const_cast < RepeatedField * > ( this ) - > unsafe_elements ( is_soo ) ;
}
// Returns a pointer to the HeapRep struct.
// pre-condition: the HeapRep must have been allocated, ie elements() is safe.
// pre-condition: the HeapRep must have been allocated, ie !is_soo() .
HeapRep * heap_rep ( ) const {
return reinterpret_cast < HeapRep * > ( reinterpret_cast < char * > ( elements ( ) ) -
ABSL_DCHECK ( ! is_soo ( ) ) ;
return reinterpret_cast < HeapRep * > ( soo_rep_ . long_rep . elements ( ) -
kHeapRepHeaderSize ) ;
}
// Internal helper to delete all elements and deallocate the storage.
template < bool in_destructor = false >
void InternalDeallocate ( ) {
const size_t bytes = Capacity ( ) * sizeof ( Element ) + kHeapRepHeaderSize ;
ABSL_DCHECK ( ! is_soo ( ) ) ;
const size_t bytes = Capacity ( false ) * sizeof ( Element ) + kHeapRepHeaderSize ;
if ( heap_rep ( ) - > arena = = nullptr ) {
internal : : SizedDelete ( heap_rep ( ) , bytes ) ;
} else if ( ! in_destructor ) {
@ -468,62 +621,70 @@ class RepeatedField final
// empty (common case), and add only an 8-byte header to the elements array
// when non-empty. We make sure to place the size fields directly in the
// RepeatedField class to avoid costly cache misses due to the indirection.
int size_ ;
int capacity_ ;
// If capacity_ == 0 this points to an Arena otherwise it points to the
// elements member of a HeapRep struct. Using this invariant allows the
// storage of the arena pointer without an extra allocation in the
// constructor.
void * arena_or_elements_ ;
internal : : SooRep soo_rep_ { } ;
} ;
// implementation ====================================================
template < typename Element >
constexpr RepeatedField < Element > : : RepeatedField ( )
: size_ ( 0 ) , capacity_ ( 0 ) , arena_or_elements_ ( nullptr ) {
constexpr RepeatedField < Element > : : RepeatedField ( ) {
StaticValidityCheck ( ) ;
# ifdef __cpp_lib_is_constant_evaluated
if ( ! std : : is_constant_evaluated ( ) ) {
AnnotateSize ( kSooCapacityElements , 0 ) ;
}
# endif // __cpp_lib_is_constant_evaluated
}
template < typename Element >
inline RepeatedField < Element > : : RepeatedField ( Arena * arena )
: size_ ( 0 ) , capacity_ ( 0 ) , arena_or_elements_ ( arena ) {
inline RepeatedField < Element > : : RepeatedField ( Arena * arena ) : soo_rep_ ( arena ) {
StaticValidityCheck ( ) ;
AnnotateSize ( kSooCapacityElements , 0 ) ;
}
template < typename Element >
inline RepeatedField < Element > : : RepeatedField ( Arena * arena ,
const RepeatedField & rhs )
: size_ ( 0 ) , capacity_ ( 0 ) , arena_or_elements _ ( arena ) {
: soo_rep _ ( arena ) {
StaticValidityCheck ( ) ;
if ( auto size = rhs . size ( ) ) {
Grow ( 0 , size ) ;
ExchangeCurrentSize ( size ) ;
UninitializedCopyN ( rhs . elements ( ) , size , unsafe_elements ( ) ) ;
AnnotateSize ( kSooCapacityElements , 0 ) ;
const bool rhs_is_soo = rhs . is_soo ( ) ;
if ( auto size = rhs . size ( rhs_is_soo ) ) {
bool is_soo = true ;
if ( size > kSooCapacityElements ) {
Grow ( is_soo , 0 , size ) ;
is_soo = false ;
}
ExchangeCurrentSize ( is_soo , size ) ;
UninitializedCopyN ( rhs . elements ( rhs_is_soo ) , size , unsafe_elements ( is_soo ) ) ;
}
}
template < typename Element >
template < typename Iter , typename >
RepeatedField < Element > : : RepeatedField ( Iter begin , Iter end )
: size_ ( 0 ) , capacity_ ( 0 ) , arena_or_elements_ ( nullptr ) {
RepeatedField < Element > : : RepeatedField ( Iter begin , Iter end ) {
StaticValidityCheck ( ) ;
AnnotateSize ( kSooCapacityElements , 0 ) ;
Add ( begin , end ) ;
}
template < typename Element >
RepeatedField < Element > : : ~ RepeatedField ( ) {
StaticValidityCheck ( ) ;
const bool is_soo = this - > is_soo ( ) ;
# ifndef NDEBUG
// Try to trigger segfault / asan failure in non-opt builds if arena_
// lifetime has ended before the destructor.
auto arena = GetArena ( ) ;
auto arena = GetArena ( is_soo ) ;
if ( arena ) ( void ) arena - > SpaceAllocated ( ) ;
# endif
if ( Capacity ( ) > 0 ) {
Destroy ( unsafe_elements ( ) , unsafe_elements ( ) + size ( ) ) ;
InternalDeallocate < true > ( ) ;
const int size = this - > size ( is_soo ) ;
if ( size > 0 ) {
Element * elem = unsafe_elements ( is_soo ) ;
Destroy ( elem , elem + size ) ;
}
UnpoisonBuffer ( ) ;
if ( ! is_soo ) InternalDeallocate < true > ( ) ;
}
template < typename Element >
@ -555,9 +716,10 @@ inline RepeatedField<Element>& RepeatedField<Element>::operator=(
// We don't just call Swap(&other) here because it would perform 3 copies if
// the two fields are on different arenas.
if ( this ! = & other ) {
if ( GetArena ( ) ! = other . GetArena ( )
const Arena * arena = GetArena ( ) ;
if ( arena ! = other . GetArena ( )
# ifdef PROTOBUF_FORCE_COPY_IN_MOVE
| | GetArena ( ) = = nullptr
| | arena = = nullptr
# endif // !PROTOBUF_FORCE_COPY_IN_MOVE
) {
CopyFrom ( other ) ;
@ -575,36 +737,44 @@ inline bool RepeatedField<Element>::empty() const {
template < typename Element >
inline int RepeatedField < Element > : : size ( ) const {
return size_ ;
return size ( is _soo ( ) ) ;
}
template < typename Element >
inline int RepeatedField < Element > : : Capacity ( ) const {
return capacity_ ;
return Capacity ( is_soo ( ) ) ;
}
template < typename Element >
inline void RepeatedField < Element > : : AddAlreadyReserved ( Element value ) {
ABSL_DCHECK_LT ( size ( ) , Capacity ( ) ) ;
void * p = elements ( ) + ExchangeCurrentSize ( size ( ) + 1 ) ;
const bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
ABSL_DCHECK_LT ( old_size , Capacity ( is_soo ) ) ;
void * p = elements ( is_soo ) + ExchangeCurrentSize ( is_soo , old_size + 1 ) ;
: : new ( p ) Element ( std : : move ( value ) ) ;
}
template < typename Element >
inline Element * RepeatedField < Element > : : AddAlreadyReserved ( )
ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_DCHECK_LT ( size ( ) , Capacity ( ) ) ;
const bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
ABSL_DCHECK_LT ( old_size , Capacity ( is_soo ) ) ;
// new (p) <TrivialType> compiles into nothing: this is intentional as this
// function is documented to return uninitialized data for trivial types.
void * p = elements ( ) + ExchangeCurrentSize ( size ( ) + 1 ) ;
void * p = elements ( is_soo ) + ExchangeCurrentSize ( is_soo , old_size + 1 ) ;
return : : new ( p ) Element ;
}
template < typename Element >
inline Element * RepeatedField < Element > : : AddNAlreadyReserved ( int n )
ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_DCHECK_GE ( Capacity ( ) - size ( ) , n ) < < Capacity ( ) < < " , " < < size ( ) ;
Element * p = unsafe_elements ( ) + ExchangeCurrentSize ( size ( ) + n ) ;
const bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
ABSL_ATTRIBUTE_UNUSED const int capacity = Capacity ( is_soo ) ;
ABSL_DCHECK_GE ( capacity - old_size , n ) < < capacity < < " , " < < old_size ;
Element * p =
unsafe_elements ( is_soo ) + ExchangeCurrentSize ( is_soo , old_size + n ) ;
for ( Element * begin = p , * end = p + n ; begin ! = end ; + + begin ) {
new ( static_cast < void * > ( begin ) ) Element ;
}
@ -614,15 +784,20 @@ inline Element* RepeatedField<Element>::AddNAlreadyReserved(int n)
template < typename Element >
inline void RepeatedField < Element > : : Resize ( int new_size , const Element & value ) {
ABSL_DCHECK_GE ( new_size , 0 ) ;
const int old_size = size ( ) ;
bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
if ( new_size > old_size ) {
if ( new_size > Capacity ( ) ) Grow ( old_size , new_size ) ;
Element * first = elements ( ) + ExchangeCurrentSize ( new_size ) ;
std : : uninitialized_fill ( first , elements ( ) + new_size , value ) ;
if ( new_size > Capacity ( is_soo ) ) {
Grow ( is_soo , old_size , new_size ) ;
is_soo = false ;
}
Element * elem = elements ( is_soo ) ;
Element * first = elem + ExchangeCurrentSize ( is_soo , new_size ) ;
std : : uninitialized_fill ( first , elem + new_size , value ) ;
} else if ( new_size < old_size ) {
Element * elem = unsafe_elements ( ) ;
Element * elem = unsafe_elements ( is_soo ) ;
Destroy ( elem + new_size , elem + old_size ) ;
ExchangeCurrentSize ( new_size ) ;
ExchangeCurrentSize ( is_soo , new_size ) ;
}
}
@ -631,7 +806,7 @@ inline const Element& RepeatedField<Element>::Get(int index) const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_DCHECK_GE ( index , 0 ) ;
ABSL_DCHECK_LT ( index , size ( ) ) ;
return elements ( ) [ index ] ;
return elements ( is_soo ( ) ) [ index ] ;
}
template < typename Element >
@ -639,7 +814,7 @@ inline const Element& RepeatedField<Element>::at(int index) const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_CHECK_GE ( index , 0 ) ;
ABSL_CHECK_LT ( index , size ( ) ) ;
return elements ( ) [ index ] ;
return elements ( is_soo ( ) ) [ index ] ;
}
template < typename Element >
@ -647,7 +822,7 @@ inline Element& RepeatedField<Element>::at(int index)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_CHECK_GE ( index , 0 ) ;
ABSL_CHECK_LT ( index , size ( ) ) ;
return elements ( ) [ index ] ;
return elements ( is_soo ( ) ) [ index ] ;
}
template < typename Element >
@ -655,7 +830,7 @@ inline Element* RepeatedField<Element>::Mutable(int index)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_DCHECK_GE ( index , 0 ) ;
ABSL_DCHECK_LT ( index , size ( ) ) ;
return & elements ( ) [ index ] ;
return & elements ( is_soo ( ) ) [ index ] ;
}
template < typename Element >
@ -665,69 +840,92 @@ inline void RepeatedField<Element>::Set(int index, const Element& value) {
template < typename Element >
inline void RepeatedField < Element > : : Add ( Element value ) {
const int old_size = size ( ) ;
int capacity = Capacity ( ) ;
Element * elem = unsafe_elements ( ) ;
bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
int capacity = Capacity ( is_soo ) ;
Element * elem = unsafe_elements ( is_soo ) ;
if ( ABSL_PREDICT_FALSE ( old_size = = capacity ) ) {
Grow ( old_size , old_size + 1 ) ;
capacity = Capacity ( ) ;
elem = unsafe_elements ( ) ;
Grow ( is_soo , old_size , old_size + 1 ) ;
is_soo = false ;
capacity = Capacity ( is_soo ) ;
elem = unsafe_elements ( is_soo ) ;
}
int new_size = old_size + 1 ;
void * p = elem + ExchangeCurrentSize ( new_size ) ;
void * p = elem + ExchangeCurrentSize ( is_soo , new_size ) ;
: : new ( p ) Element ( std : : move ( value ) ) ;
// The below helps the compiler optimize dense loops.
ABSL_ASSUME ( new_size = = size_ ) ;
ABSL_ASSUME ( elem = = arena_or_elements_ ) ;
ABSL_ASSUME ( capacity = = capacity_ ) ;
// Note: we can't call functions in PROTOBUF_ASSUME so use local variables.
ABSL_ATTRIBUTE_UNUSED const int final_is_soo = this - > is_soo ( ) ;
PROTOBUF_ASSUME ( is_soo = = final_is_soo ) ;
ABSL_ATTRIBUTE_UNUSED const int final_size = size ( is_soo ) ;
PROTOBUF_ASSUME ( new_size = = final_size ) ;
ABSL_ATTRIBUTE_UNUSED Element * const final_elements = unsafe_elements ( is_soo ) ;
PROTOBUF_ASSUME ( elem = = final_elements ) ;
ABSL_ATTRIBUTE_UNUSED const int final_capacity = Capacity ( is_soo ) ;
PROTOBUF_ASSUME ( capacity = = final_capacity ) ;
}
template < typename Element >
inline Element * RepeatedField < Element > : : Add ( ) ABSL_ATTRIBUTE_LIFETIME_BOUND {
const int old_size = size ( ) ;
bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
if ( ABSL_PREDICT_FALSE ( old_size = = Capacity ( ) ) ) {
Grow ( old_size , old_size + 1 ) ;
Grow ( is_soo , old_size , old_size + 1 ) ;
is_soo = false ;
}
void * p = unsafe_elements ( ) + ExchangeCurrentSize ( old_size + 1 ) ;
void * p = unsafe_elements ( is_soo ) + ExchangeCurrentSize ( is_soo , old_size + 1 ) ;
return : : new ( p ) Element ;
}
template < typename Element >
template < typename Iter >
inline void RepeatedField < Element > : : AddForwardIterator ( Iter begin , Iter end ) {
const int old_size = size ( ) ;
int capacity = Capacity ( ) ;
Element * elem = unsafe_elements ( ) ;
bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
int capacity = Capacity ( is_soo ) ;
Element * elem = unsafe_elements ( is_soo ) ;
int new_size = old_size + static_cast < int > ( std : : distance ( begin , end ) ) ;
if ( ABSL_PREDICT_FALSE ( new_size > capacity ) ) {
Grow ( old_size , new_size ) ;
elem = unsafe_elements ( ) ;
capacity = Capacity ( ) ;
Grow ( is_soo , old_size , new_size ) ;
is_soo = false ;
elem = unsafe_elements ( is_soo ) ;
capacity = Capacity ( is_soo ) ;
}
UninitializedCopy ( begin , end , elem + ExchangeCurrentSize ( new_size ) ) ;
UninitializedCopy ( begin , end , elem + ExchangeCurrentSize ( is_soo , new_size ) ) ;
// The below helps the compiler optimize dense loops.
ABSL_ASSUME ( new_size = = size_ ) ;
ABSL_ASSUME ( elem = = arena_or_elements_ ) ;
ABSL_ASSUME ( capacity = = capacity_ ) ;
// Note: we can't call functions in PROTOBUF_ASSUME so use local variables.
ABSL_ATTRIBUTE_UNUSED const int final_is_soo = this - > is_soo ( ) ;
PROTOBUF_ASSUME ( is_soo = = final_is_soo ) ;
ABSL_ATTRIBUTE_UNUSED const int final_size = size ( is_soo ) ;
PROTOBUF_ASSUME ( new_size = = final_size ) ;
ABSL_ATTRIBUTE_UNUSED Element * const final_elements = unsafe_elements ( is_soo ) ;
PROTOBUF_ASSUME ( elem = = final_elements ) ;
ABSL_ATTRIBUTE_UNUSED const int final_capacity = Capacity ( is_soo ) ;
PROTOBUF_ASSUME ( capacity = = final_capacity ) ;
}
template < typename Element >
template < typename Iter >
inline void RepeatedField < Element > : : AddInputIterator ( Iter begin , Iter end ) {
Element * elem = unsafe_elements ( ) ;
Element * first = elem + size ( ) ;
Element * last = elem + Capacity ( ) ;
AnnotateSize ( size ( ) , Capacity ( ) ) ;
bool is_soo = this - > is_soo ( ) ;
int size = this - > size ( is_soo ) ;
int capacity = Capacity ( is_soo ) ;
Element * elem = unsafe_elements ( is_soo ) ;
Element * first = elem + size ;
Element * last = elem + capacity ;
UnpoisonBuffer ( ) ;
while ( begin ! = end ) {
if ( ABSL_PREDICT_FALSE ( first = = last ) ) {
int size = first - elem ;
GrowNoAnnotate ( size , size + 1 ) ;
elem = unsafe_elements ( ) ;
size = first - elem ;
GrowNoAnnotate ( is_soo , size , size + 1 ) ;
is_soo = false ;
elem = unsafe_elements ( is_soo ) ;
capacity = Capacity ( is_soo ) ;
first = elem + size ;
last = elem + Capacity ( ) ;
last = elem + capacity ;
}
: : new ( static_cast < void * > ( first ) ) Element ( * begin ) ;
+ + begin ;
@ -735,8 +933,8 @@ inline void RepeatedField<Element>::AddInputIterator(Iter begin, Iter end) {
}
const int new_size = first - elem ;
set_size ( new_size ) ;
AnnotateSize ( Capacity ( ) , new_size ) ;
set_size ( is_soo , new_size ) ;
AnnotateSize ( capacity , new_size ) ;
}
template < typename Element >
@ -753,10 +951,11 @@ inline void RepeatedField<Element>::Add(Iter begin, Iter end) {
template < typename Element >
inline void RepeatedField < Element > : : RemoveLast ( ) {
const int old_size = size ( ) ;
const bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
ABSL_DCHECK_GT ( old_size , 0 ) ;
elements ( ) [ old_size - 1 ] . ~ Element ( ) ;
ExchangeCurrentSize ( old_size - 1 ) ;
elements ( is_soo ) [ old_size - 1 ] . ~ Element ( ) ;
ExchangeCurrentSize ( is_soo , old_size - 1 ) ;
}
template < typename Element >
@ -764,9 +963,10 @@ void RepeatedField<Element>::ExtractSubrange(int start, int num,
Element * elements ) {
ABSL_DCHECK_GE ( start , 0 ) ;
ABSL_DCHECK_GE ( num , 0 ) ;
const int old_size = size ( ) ;
const bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
ABSL_DCHECK_LE ( start + num , old_size ) ;
Element * elem = unsafe_elements ( ) ;
Element * elem = unsafe_elements ( is_soo ) ;
// Save the values of the removed elements if requested.
if ( elements ! = nullptr ) {
@ -783,19 +983,23 @@ void RepeatedField<Element>::ExtractSubrange(int start, int num,
template < typename Element >
inline void RepeatedField < Element > : : Clear ( ) {
Element * elem = unsafe_elements ( ) ;
Destroy ( elem , elem + size ( ) ) ;
ExchangeCurrentSize ( 0 ) ;
const bool is_soo = this - > is_soo ( ) ;
Element * elem = unsafe_elements ( is_soo ) ;
Destroy ( elem , elem + size ( is_soo ) ) ;
ExchangeCurrentSize ( is_soo , 0 ) ;
}
template < typename Element >
inline void RepeatedField < Element > : : MergeFrom ( const RepeatedField & other ) {
ABSL_DCHECK_NE ( & other , this ) ;
if ( auto other_size = other . size ( ) ) {
const bool other_is_soo = other . is_soo ( ) ;
if ( auto other_size = other . size ( other_is_soo ) ) {
const int old_size = size ( ) ;
Reserve ( old_size + other_size ) ;
Element * dst = elements ( ) + ExchangeCurrentSize ( old_size + other_size ) ;
UninitializedCopyN ( other . elements ( ) , other_size , dst ) ;
const bool is_soo = this - > is_soo ( ) ;
Element * dst =
elements ( is_soo ) + ExchangeCurrentSize ( is_soo , old_size + other_size ) ;
UninitializedCopyN ( other . elements ( other_is_soo ) , other_size , dst ) ;
}
}
@ -832,13 +1036,13 @@ inline typename RepeatedField<Element>::iterator RepeatedField<Element>::erase(
template < typename Element >
inline Element * RepeatedField < Element > : : mutable_data ( )
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return unsafe_elements ( ) ;
return unsafe_elements ( is_soo ( ) ) ;
}
template < typename Element >
inline const Element * RepeatedField < Element > : : data ( ) const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return unsafe_elements ( ) ;
return unsafe_elements ( is_soo ( ) ) ;
}
template < typename Element >
@ -846,27 +1050,31 @@ inline void RepeatedField<Element>::InternalSwap(
RepeatedField * PROTOBUF_RESTRICT other ) {
ABSL_DCHECK ( this ! = other ) ;
// Swap all fields at once.
static_assert ( std : : is_standard_layout < RepeatedField < Element > > : : value ,
" offsetof() requires standard layout before c++17 " ) ;
static constexpr size_t kOffset = offsetof ( RepeatedField , size_ ) ;
internal : : memswap < offsetof ( RepeatedField , arena_or_elements_ ) +
sizeof ( this - > arena_or_elements_ ) - kOffset > (
reinterpret_cast < char * > ( this ) + kOffset ,
reinterpret_cast < char * > ( other ) + kOffset ) ;
// We need to unpoison during the swap in case we're in SOO mode.
UnpoisonBuffer ( ) ;
other - > UnpoisonBuffer ( ) ;
internal : : memswap < sizeof ( internal : : SooRep ) > (
reinterpret_cast < char * > ( & this - > soo_rep_ ) ,
reinterpret_cast < char * > ( & other - > soo_rep_ ) ) ;
AnnotateSize ( Capacity ( ) , size ( ) ) ;
other - > AnnotateSize ( other - > Capacity ( ) , other - > size ( ) ) ;
}
template < typename Element >
void RepeatedField < Element > : : Swap ( RepeatedField * other ) {
if ( this = = other ) return ;
Arena * arena = GetArena ( ) ;
Arena * other_arena = other - > GetArena ( ) ;
# ifdef PROTOBUF_FORCE_COPY_IN_SWAP
if ( GetArena ( ) ! = nullptr & & GetArena ( ) = = other - > GetArena ( ) ) {
if ( arena ! = nullptr & & arena = = other_arena ) {
# else // PROTOBUF_FORCE_COPY_IN_SWAP
if ( GetArena ( ) = = other - > GetArena ( ) ) {
if ( arena = = other_arena ) {
# endif // !PROTOBUF_FORCE_COPY_IN_SWAP
InternalSwap ( other ) ;
} else {
RepeatedField < Element > temp ( other - > GetArena ( ) ) ;
RepeatedField < Element > temp ( other_arena ) ;
temp . MergeFrom ( * this ) ;
CopyFrom ( * other ) ;
other - > UnsafeArenaSwap ( & temp ) ;
@ -882,45 +1090,51 @@ void RepeatedField<Element>::UnsafeArenaSwap(RepeatedField* other) {
template < typename Element >
void RepeatedField < Element > : : SwapElements ( int index1 , int index2 ) {
Element * elem = elements ( is_soo ( ) ) ;
using std : : swap ; // enable ADL with fallback
swap ( elements ( ) [ index1 ] , elements ( ) [ index2 ] ) ;
swap ( elem [ index1 ] , elem [ index2 ] ) ;
}
template < typename Element >
inline typename RepeatedField < Element > : : iterator RepeatedField < Element > : : begin ( )
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return iterator ( unsafe_elements ( ) ) ;
return iterator ( unsafe_elements ( is_soo ( ) ) ) ;
}
template < typename Element >
inline typename RepeatedField < Element > : : const_iterator
RepeatedField < Element > : : begin ( ) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_iterator ( unsafe_elements ( ) ) ;
return const_iterator ( unsafe_elements ( is_soo ( ) ) ) ;
}
template < typename Element >
inline typename RepeatedField < Element > : : const_iterator
RepeatedField < Element > : : cbegin ( ) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_iterator ( unsafe_elements ( ) ) ;
return const_iterator ( unsafe_elements ( is_soo ( ) ) ) ;
}
template < typename Element >
inline typename RepeatedField < Element > : : iterator RepeatedField < Element > : : end ( )
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return iterator ( unsafe_elements ( ) + size ( ) ) ;
const bool is_soo = this - > is_soo ( ) ;
return iterator ( unsafe_elements ( is_soo ) + size ( is_soo ) ) ;
}
template < typename Element >
inline typename RepeatedField < Element > : : const_iterator
RepeatedField < Element > : : end ( ) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_iterator ( unsafe_elements ( ) + size ( ) ) ;
const bool is_soo = this - > is_soo ( ) ;
return const_iterator ( unsafe_elements ( is_soo ) + size ( is_soo ) ) ;
}
template < typename Element >
inline typename RepeatedField < Element > : : const_iterator
RepeatedField < Element > : : cend ( ) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_iterator ( unsafe_elements ( ) + size ( ) ) ;
const bool is_soo = this - > is_soo ( ) ;
return const_iterator ( unsafe_elements ( is_soo ) + size ( is_soo ) ) ;
}
template < typename Element >
inline size_t RepeatedField < Element > : : SpaceUsedExcludingSelfLong ( ) const {
const int capacity = Capacity ( ) ;
return capacity > 0 ? capacity * sizeof ( Element ) + kHeapRepHeaderSize : 0 ;
return capacity > kSooCapacityElements
? capacity * sizeof ( Element ) + kHeapRepHeaderSize
: 0 ;
}
namespace internal {
@ -928,10 +1142,7 @@ namespace internal {
// requested 'new_size'. The result is clamped to the closed interval:
// [internal::kMinRepeatedFieldAllocationSize,
// std::numeric_limits<int>::max()]
// Requires:
// new_size > capacity &&
// (capacity == 0 ||
// capacity >= kRepeatedFieldLowerClampLimit)
// Requires: new_size > capacity
template < typename T , int kHeapRepHeaderSize >
inline int CalculateReserveSize ( int capacity , int new_size ) {
constexpr int lower_limit =
@ -945,6 +1156,15 @@ inline int CalculateReserveSize(int capacity, int new_size) {
if ( PROTOBUF_PREDICT_FALSE ( capacity > kMaxSizeBeforeClamp ) ) {
return std : : numeric_limits < int > : : max ( ) ;
}
constexpr int kSooCapacityElements = SooCapacityElements ( sizeof ( T ) ) ;
if ( kSooCapacityElements > 0 & & kSooCapacityElements < lower_limit ) {
// In this case, we need to set capacity to 0 here to ensure power-of-two
// sized allocations.
if ( capacity < lower_limit ) capacity = 0 ;
} else {
ABSL_DCHECK ( capacity = = 0 | | capacity > = lower_limit )
< < capacity < < " " < < lower_limit ;
}
// We want to double the number of bytes, not the number of elements, to try
// to stay within power-of-two allocations.
// The allocation has kHeapRepHeaderSize + sizeof(T) * capacity.
@ -955,22 +1175,25 @@ inline int CalculateReserveSize(int capacity, int new_size) {
template < typename Element >
void RepeatedField < Element > : : Reserve ( int new_size ) {
if ( ABSL_PREDICT_FALSE ( new_size > Capacity ( ) ) ) {
Grow ( size ( ) , new_size ) ;
const bool was_soo = is_soo ( ) ;
if ( ABSL_PREDICT_FALSE ( new_size > Capacity ( was_soo ) ) ) {
Grow ( was_soo , size ( was_soo ) , new_size ) ;
}
}
// Avoid inlining of Reserve(): new, copy, and delete[] lead to a significant
// amount of code bloat.
template < typename Element >
PROTOBUF_NOINLINE void RepeatedField < Element > : : GrowNoAnnotate ( int old_size ,
PROTOBUF_NOINLINE void RepeatedField < Element > : : GrowNoAnnotate ( bool was_soo ,
int old_size ,
int new_size ) {
ABSL_DCHECK_GT ( new_size , Capacity ( ) ) ;
const int old_capacity = Capacity ( was_soo ) ;
ABSL_DCHECK_GT ( new_size , old_capacity ) ;
HeapRep * new_rep ;
Arena * arena = GetArena ( ) ;
new_size = internal : : CalculateReserveSize < Element , kHeapRepHeaderSize > (
Capacity ( ) , new_size ) ;
old_capacity , new_size ) ;
ABSL_DCHECK_LE ( static_cast < size_t > ( new_size ) ,
( std : : numeric_limits < size_t > : : max ( ) - kHeapRepHeaderSize ) /
@ -994,25 +1217,22 @@ PROTOBUF_NOINLINE void RepeatedField<Element>::GrowNoAnnotate(int old_size,
}
new_rep - > arena = arena ;
if ( Capacity ( ) > 0 ) {
if ( old_size > 0 ) {
Element * pnew = static_cast < Element * > ( new_rep - > elements ( ) ) ;
Element * pold = elements ( ) ;
// TODO: add absl::is_trivially_relocatable<Element>
if ( std : : is_trivial < Element > : : value ) {
memcpy ( static_cast < void * > ( pnew ) , pold , old_size * sizeof ( Element ) ) ;
} else {
for ( Element * end = pnew + old_size ; pnew ! = end ; + + pnew , + + pold ) {
: : new ( static_cast < void * > ( pnew ) ) Element ( std : : move ( * pold ) ) ;
pold - > ~ Element ( ) ;
}
if ( old_size > 0 ) {
Element * pnew = static_cast < Element * > ( new_rep - > elements ( ) ) ;
Element * pold = elements ( was_soo ) ;
// TODO: add absl::is_trivially_relocatable<Element>
if ( std : : is_trivial < Element > : : value ) {
memcpy ( static_cast < void * > ( pnew ) , pold , old_size * sizeof ( Element ) ) ;
} else {
for ( Element * end = pnew + old_size ; pnew ! = end ; + + pnew , + + pold ) {
: : new ( static_cast < void * > ( pnew ) ) Element ( std : : move ( * pold ) ) ;
pold - > ~ Element ( ) ;
}
}
InternalDeallocate ( ) ;
}
if ( ! was_soo ) InternalDeallocate ( ) ;
set_capacity ( new_size ) ;
arena_or_elements_ = static_cast < Element * > ( new_rep - > elements ( ) ) ;
soo_rep_ . set_non_soo ( was_soo , new_size , new_rep - > elements ( ) ) ;
}
// Ideally we would be able to use:
@ -1021,21 +1241,22 @@ PROTOBUF_NOINLINE void RepeatedField<Element>::GrowNoAnnotate(int old_size,
// However, as explained in b/266411038#comment9, this causes issues
// in shared libraries for Youtube (and possibly elsewhere).
template < typename Element >
PROTOBUF_NOINLINE void RepeatedField < Element > : : Grow ( int old_size ,
PROTOBUF_NOINLINE void RepeatedField < Element > : : Grow ( bool was_soo , int old_size ,
int new_size ) {
AnnotateSize ( old_size , Capacity ( ) ) ;
GrowNoAnnotate ( old_size , new_size ) ;
UnpoisonBuffer ( ) ;
GrowNoAnnotate ( was_soo , old_size , new_size ) ;
AnnotateSize ( Capacity ( ) , old_size ) ;
}
template < typename Element >
inline void RepeatedField < Element > : : Truncate ( int new_size ) {
const int old_size = size ( ) ;
const bool is_soo = this - > is_soo ( ) ;
const int old_size = size ( is_soo ) ;
ABSL_DCHECK_LE ( new_size , old_size ) ;
if ( new_size < old_size ) {
Element * elem = unsafe_elements ( ) ;
Element * elem = unsafe_elements ( is_soo ) ;
Destroy ( elem + new_size , elem + old_size ) ;
ExchangeCurrentSize ( new_size ) ;
ExchangeCurrentSize ( is_soo , new_size ) ;
}
}