// Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd // This file defines an Arena allocator for better allocation performance. #ifndef GOOGLE_PROTOBUF_ARENA_H__ #define GOOGLE_PROTOBUF_ARENA_H__ #include #include #include #include #include // IWYU pragma: keep for operator new(). #include #include #include #include #include "absl/base/macros.h" #include "google/protobuf/internal_visibility.h" #if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS // Work around bugs in MSVC header when _HAS_EXCEPTIONS=0. #include #include namespace std { using type_info = ::type_info; } #endif #include "absl/base/attributes.h" #include "absl/base/optimization.h" #include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/strings/str_format.h" #include "google/protobuf/arena_align.h" #include "google/protobuf/arena_allocation_policy.h" #include "google/protobuf/port.h" #include "google/protobuf/serial_arena.h" #include "google/protobuf/thread_safe_arena.h" // Must be included last. #include "google/protobuf/port_def.inc" #ifdef SWIG #error "You cannot SWIG proto headers" #endif namespace google { namespace protobuf { struct ArenaOptions; // defined below class Arena; // defined below class Message; // defined in message.h class MessageLite; template class Map; namespace internal { class RepeatedFieldBase; class ExtensionSet; } // namespace internal namespace TestUtil { class ReflectionTester; // defined in test_util.h } // namespace TestUtil namespace internal { struct ArenaTestPeer; // defined in arena_test_util.h class InternalMetadata; // defined in metadata_lite.h class LazyField; // defined in lazy_field.h class EpsCopyInputStream; // defined in parse_context.h class UntypedMapBase; // defined in map.h class RepeatedPtrFieldBase; // defined in repeated_ptr_field.h class TcParser; // defined in generated_message_tctable_impl.h SerialArena* PROTOBUF_NULLABLE GetSerialArena(Arena* PROTOBUF_NULLABLE); template class GenericTypeHandler; // defined in repeated_field.h // This struct maps field types to the types that we will use to represent them // when allocated on an arena. This is necessary because fields no longer own an // arena pointer, but can be allocated directly on an arena. In this case, we // will use a wrapper class that holds both the arena pointer and the field, and // points the field to the arena pointer. // // Additionally, split pointer fields will use this representation when // allocated, regardless of whether they are on an arena or not. // // For example: // ``` // template <> // struct FieldArenaRep { // using Type = ArenaMessage; // static Message* Get(ArenaMessage* arena_rep) { // return &arena_rep->message(); // } // }; // ``` template struct FieldArenaRep { // The type of the field when allocated on an arena. By default, this is just // `T`, but can be specialized to use a wrapper class that holds both the // arena pointer and the field. using Type = T; // Returns a pointer to the field from the arena representation. By default, // this is just a no-op, but can be specialized to extract the field from the // wrapper class. static T* PROTOBUF_NONNULL Get(Type* PROTOBUF_NONNULL arena_rep) { return arena_rep; } }; // Returns true if `T` uses arena offsets instead of holding a copy of the arena // pointer. This can be deduced if the field's arena representation is not the // same as the field itself. template constexpr bool FieldHasArenaOffset() { using ArenaRepT = typename FieldArenaRep::Type; return !std::is_same_v; } // TODO - Some types have a deprecated arena-enabled constructor, // as we plan to remove it in favor of using arena offsets, but for now Arena // needs to call it. While the arena constructor exists, we will call the // `InternalVisibility` override to silence the warning. template constexpr bool HasDeprecatedArenaConstructor() { return std::is_base_of_v && !std::is_same_v; } template void arena_delete_object(void* PROTOBUF_NONNULL object) { delete reinterpret_cast(object); } inline bool CanUseInternalSwap(Arena* PROTOBUF_NULLABLE lhs, Arena* PROTOBUF_NULLABLE rhs) { if (DebugHardenForceCopyInSwap()) { // We force copy in swap when we are not using an arena. // If we did with an arena we would grow arena usage too much. return lhs != nullptr && lhs == rhs; } else { return lhs == rhs; } } inline bool CanMoveWithInternalSwap(Arena* PROTOBUF_NULLABLE lhs, Arena* PROTOBUF_NULLABLE rhs) { if (DebugHardenForceCopyInMove()) { // We force copy in move when we are not using an arena. // If we did with an arena we would grow arena usage too much. return lhs != nullptr && lhs == rhs; } else { return lhs == rhs; } } } // namespace internal // ArenaOptions provides optional additional parameters to arena construction // that control its block-allocation behavior. struct ABSL_ATTRIBUTE_WARN_UNUSED ArenaOptions final { // This defines the size of the first block requested from the system malloc. // Subsequent block sizes will increase in a geometric series up to a maximum. size_t start_block_size = internal::AllocationPolicy::kDefaultStartBlockSize; // This defines the maximum block size requested from system malloc (unless an // individual arena allocation request occurs with a size larger than this // maximum). Requested block sizes increase up to this value, then remain // here. size_t max_block_size = internal::AllocationPolicy::DefaultMaxBlockSize(); // An initial block of memory for the arena to use, or nullptr for none. If // provided, the block must live at least as long as the arena itself. The // creator of the Arena retains ownership of the block after the Arena is // destroyed. char* PROTOBUF_NULLABLE initial_block = nullptr; // The size of the initial block, if provided. size_t initial_block_size = 0; // A function pointer to an alloc method that returns memory blocks of size // requested. By default, it contains a ptr to the malloc function. // // NOTE: block_alloc and dealloc functions are expected to behave like // malloc and free, including Asan poisoning. void* PROTOBUF_NONNULL (*PROTOBUF_NULLABLE block_alloc)(size_t) = nullptr; // A function pointer to a dealloc method that takes ownership of the blocks // from the arena. By default, it contains a ptr to a wrapper function that // calls free. void (*PROTOBUF_NULLABLE block_dealloc)(void* PROTOBUF_NONNULL, size_t) = nullptr; private: internal::AllocationPolicy AllocationPolicy() const { internal::AllocationPolicy res; res.start_block_size = start_block_size; res.max_block_size = max_block_size; res.block_alloc = block_alloc; res.block_dealloc = block_dealloc; return res; } friend class Arena; friend class ArenaOptionsTestFriend; }; // Arena allocator. Arena allocation replaces ordinary (heap-based) allocation // with new/delete, and improves performance by aggregating allocations into // larger blocks and freeing allocations all at once. Protocol messages are // allocated on an arena by using Arena::Create(Arena*), below, and are // automatically freed when the arena is destroyed. // // This is a thread-safe implementation: multiple threads may allocate from the // arena concurrently. Destruction is not thread-safe and the destructing // thread must synchronize with users of the arena first. class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) #ifdef __clang__ // TODO: Enable this for GCC. ABSL_ATTRIBUTE_WARN_UNUSED #endif // __clang__ Arena final { public: // A unique-pointer-like smart pointer type for holding objects that // correctly and safely deletes them, whether or not the objects owned by // protobuf `Arena`s. `UniquePtr` is used to hold either a newly created // object or a message released from a parent container. // // In spirit, an `Arena::UniquePtr` is akin to // `std::variant, Arena::Ptr>`. It might semantically // contain either of those types, it can be constructed from them, and you can // extract them out as needed. // // To create an `UniquePtr`, use the helper functions in `Arena` or release a // message from a parent using one of the release functions in // message_movers.h. // // If using heap, this smart pointer will own its object and destroy it as // needed. // // `UniquePtr` provides a similar interface to `std::unique_ptr` except that // it also provides access to the message's owning `Arena`, explicitly removes // the `reset(T*)` function (though it leaves `reset()` and `reset(nullptr)`), // and makes all constructors except the move-constructor private. Instead of // `reset(T*)` or a constructor, you should use move assignment and the // `MakeUnique`/`UnsafeWrapUniquePtr` functions. // // // Example Usage: // Arena* arena_ptr = ...; // UniquePtr parent = Arena::MakeUnique(arena_ptr); // ... // UniquePtr ptr = // google::protobuf::ReleaseMessageField<"child_field">(parent); // ... // ptr.reset(); // Will delete ChildMessage ptr if arena_ptr was nullptr. // CHECK(ptr == nullptr); // ... // ModifyChildMessage(ptr.get()); // ConsumeChildMessage(std::move(ptr)); template class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI ABSL_NULLABILITY_COMPATIBLE PROTOBUF_NULL_AFTER_MOVE UniquePtr; // A smart pointer type for holding objects that are statically known to be // owned by an `Arena`. Even though it is a smart pointer, `Ptr` does not // actually own the underlying object. // // `Ptr` exists to provide invariants in the type-system in a way that `T*` // cannot. `Ptr` is similar to `T*` except it hoolds extra static // information (the fact that it is arena owned) and extra dynamic information // (the arena that owns it). Main differences from `UniquePtr` are: // - `Ptr` is never null. It has no default state, and no moved-from state. // - `Ptr` does not own the object. The underlying `Arena` does. // - `Ptr` is copyable. Trying to move it will just copy it, just like `T*` // would. // - `Ptr` has no `reset()`. It can be assigned from another `Ptr`. template class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI Ptr; // Default constructor with sensible default options, tuned for average // use-cases. inline Arena() : impl_() {} // Construct an arena with default options, except for the supplied // initial block. It is more efficient to use this constructor // instead of passing ArenaOptions if the only configuration needed // by the caller is supplying an initial block. inline Arena(char* PROTOBUF_NULLABLE initial_block, size_t initial_block_size) : impl_(initial_block, initial_block_size) {} // Arena constructor taking custom options. See ArenaOptions above for // descriptions of the options available. explicit Arena(const ArenaOptions& options) : impl_(options.initial_block, options.initial_block_size, options.AllocationPolicy()) {} // Block overhead. Use this as a guide for how much to over-allocate the // initial block if you want an allocation of size N to fit inside it. // // WARNING: if you allocate multiple objects, it is difficult to guarantee // that a series of allocations will fit in the initial block, especially if // Arena changes its alignment guarantees in the future! static const size_t kBlockOverhead = internal::ThreadSafeArena::kBlockHeaderSize + internal::ThreadSafeArena::kSerialArenaSize; inline ~Arena() = default; // Allocates an object type T if the arena passed in is not nullptr; // otherwise, returns a heap-allocated object. // // In new code, prefer `arena.Make()` when it is statically known to have // an arena, and `Arena::MakeUnique(arena)` when you have a potentially // null Arena*. These functions return smart pointers that help manage the // lifetime of the returned object. template PROTOBUF_FUTURE_ADD_EARLY_NODISCARD PROTOBUF_NDEBUG_INLINE static T* PROTOBUF_NONNULL Create(Arena* PROTOBUF_NULLABLE arena, Args&&... args) { if constexpr (is_arena_constructable::value) { using Type = std::remove_const_t; // DefaultConstruct/CopyConstruct are optimized for messages, which // are both arena constructible and destructor skippable and they // assume much. Don't use these functions unless the invariants // hold. if constexpr (is_destructor_skippable::value) { constexpr auto construct_type = GetConstructType(); // We delegate to DefaultConstruct/CopyConstruct where appropriate // because protobuf generated classes have external templates for // these functions for code size reasons. When `if constexpr` is not // available always use the fallback. if constexpr (construct_type == ConstructType::kDefault) { return static_cast(DefaultConstruct(arena)); } else if constexpr (construct_type == ConstructType::kCopy) { return static_cast(CopyConstruct(arena, &args...)); } } return CreateArenaCompatible(arena, std::forward(args)...); } else { if (ABSL_PREDICT_FALSE(arena == nullptr)) { return new T(std::forward(args)...); } return new (arena->AllocateInternal()) T(std::forward(args)...); } } // Allocates an object type T if the arena passed in is not nullptr; // otherwise, returns a heap-allocated object. // The returned smart pointer owns the object even in the arena case. template [[nodiscard]] PROTOBUF_NDEBUG_INLINE static UniquePtr PROTOBUF_NONNULL MakeUnique(Arena* PROTOBUF_NULLABLE arena, Args&&... args) { // NOLINTNEXTLINE(google3-runtime-pointer-nullability) return UnsafeWrapUniquePtr(arena, Create(arena, std::forward(args)...)); } // Allocates an object type T in the arena. // As opposed to `MakeUnique`, this is a non-static member implying that there // is always an `Arena` instance. // The returned value is always Arena owned. // // Note that `arena->Make()` has undefined behavior if `arena` is null. If // the caller is uncertain of the nullness of the arena pointer, it should // prefer `MakeUnique(arena)` instead. template [[nodiscard]] PROTOBUF_NDEBUG_INLINE Ptr Make(Args&&... args) { return Ptr(this, Create(this, std::forward(args)...)); } // Creates a `UniquePtr` with an explicit owning arena. // // If `owning_arena` is not the actual owner of `ptr`, the behavior is // undefined. As such, this function is unsafe and should be of last resort. // // Note: The owning arena is not necessarily the same as `msg->GetArena()`. // Do not use `msg->GetArena()` as the owning arena. template [[nodiscard]] static UniquePtr PROTOBUF_NULLABLE UnsafeWrapUniquePtr( Arena* PROTOBUF_NULLABLE owning_arena, T* PROTOBUF_NULLABLE ptr) { return UniquePtr(ptr, owning_arena); } // API to delete any objects not on an arena. This can be used to safely // clean up messages or repeated fields without knowing whether or not they're // owned by an arena. The pointer passed to this function should not be used // again. template PROTOBUF_ALWAYS_INLINE static void Destroy(T* PROTOBUF_NONNULL obj) { if (InternalGetArena(obj) == nullptr) delete obj; } // Allocates memory with the specific size and alignment. PROTOBUF_FUTURE_ADD_EARLY_NODISCARD void* PROTOBUF_NONNULL AllocateAligned(size_t size, size_t align = 8) { if (align <= internal::ArenaAlignDefault::align) { return Allocate(internal::ArenaAlignDefault::Ceil(size)); } else { // We are wasting space by over allocating align - 8 bytes. Compared // to a dedicated function that takes current alignment in consideration. // Such a scheme would only waste (align - 8)/2 bytes on average, but // requires a dedicated function in the outline arena allocation // functions. Possibly re-evaluate tradeoffs later. auto align_as = internal::ArenaAlignAs(align); return align_as.Ceil(Allocate(align_as.Padded(size))); } } // Create an array of object type T on the arena *without* invoking the // constructor of T. If `arena` is null, then the return value should be freed // with `delete[] x;` (or `::operator delete[](x);`). // To ensure safe uses, this function checks at compile time // (when compiled as C++11) that T is trivially default-constructible and // trivially destructible. template PROTOBUF_FUTURE_ADD_EARLY_NODISCARD PROTOBUF_NDEBUG_INLINE static T* PROTOBUF_NONNULL CreateArray(Arena* PROTOBUF_NULLABLE arena, size_t num_elements) { static_assert(std::is_trivially_default_constructible_v, "CreateArray requires a trivially constructible type"); static_assert(std::is_trivially_destructible_v, "CreateArray requires a trivially destructible type"); ABSL_CHECK_LE(num_elements, // Max rounded down to the 8 byte alignment. (std::numeric_limits::max() & ~7) / sizeof(T)) << "Requested size is too large to fit into size_t."; if (ABSL_PREDICT_FALSE(arena == nullptr)) { return new T[num_elements]; } else { // We count on compiler to realize that if sizeof(T) is a multiple of // 8 AlignUpTo can be elided. return static_cast( arena->AllocateAlignedForArray(sizeof(T) * num_elements, alignof(T))); } } // The following routines are for monitoring. They will approximate the total // sum allocated and used memory, but the exact value is an implementation // deal. For instance allocated space depends on growth policies. Do not use // these in unit tests. Returns the total space allocated by the arena, which // is the sum of the sizes of the underlying blocks. PROTOBUF_FUTURE_ADD_EARLY_NODISCARD uint64_t SpaceAllocated() const { return impl_.SpaceAllocated(); } // Returns the total space used by the arena. Similar to SpaceAllocated but // does not include free space and block overhead. This is a best-effort // estimate and may inaccurately calculate space used by other threads // executing concurrently with the call to this method. These inaccuracies // are due to race conditions, and are bounded but unpredictable. Stale data // can lead to underestimates of the space used, and race conditions can lead // to overestimates (up to the current block size). PROTOBUF_FUTURE_ADD_EARLY_NODISCARD uint64_t SpaceUsed() const { return impl_.SpaceUsed(); } // Frees all storage allocated by this arena after calling destructors // registered with OwnDestructor() and freeing objects registered with Own(). // Any objects allocated on this arena are unusable after this call. It also // returns the total space used by the arena which is the sums of the sizes // of the allocated blocks. This method is not thread-safe. uint64_t Reset() { return impl_.Reset(); } // Adds |object| to a list of heap-allocated objects to be freed with |delete| // when the arena is destroyed or reset. template PROTOBUF_ALWAYS_INLINE void Own(T* PROTOBUF_NULLABLE object) { // Collapsing all template instantiations to one for generic Message reduces // code size, using the virtual destructor instead. using TypeToUse = std::conditional_t, MessageLite, T>; if (object != nullptr) { impl_.AddCleanup(static_cast(object), &internal::arena_delete_object); } } // Adds |object| to a list of objects whose destructors will be manually // called when the arena is destroyed or reset. This differs from Own() in // that it does not free the underlying memory with |delete|; hence, it is // normally only used for objects that are placement-newed into // arena-allocated memory. template PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* PROTOBUF_NULLABLE object) { if (object != nullptr) { impl_.AddCleanup(object, &internal::cleanup::arena_destruct_object); } } // Adds a custom member function on an object to the list of destructors that // will be manually called when the arena is destroyed or reset. This differs // from OwnDestructor() in that any member function may be specified, not only // the class destructor. PROTOBUF_ALWAYS_INLINE void OwnCustomDestructor( void* PROTOBUF_NONNULL object, void (*PROTOBUF_NONNULL destruct)(void* PROTOBUF_NONNULL)) { impl_.AddCleanup(object, destruct); } template class InternalHelper { private: // A SFINAE friendly trait that probes for `U` but always evalues to // `Arena*`. template using EnableIfArena = std::enable_if_t, Arena*>; // Use go/ranked-overloads for dispatching. struct Rank0 {}; struct Rank1 : Rank0 {}; static void InternalSwap(T* PROTOBUF_NONNULL a, T* PROTOBUF_NONNULL b) { a->InternalSwap(b); } static Arena* PROTOBUF_NULLABLE GetArena(T* PROTOBUF_NONNULL p) { return GetArena(Rank1{}, p); } template static auto GetArena(Rank1, U* PROTOBUF_NONNULL p) -> EnableIfArenaGetArena())> { return p->GetArena(); } template static Arena* PROTOBUF_NULLABLE GetArena(Rank0, U* PROTOBUF_NULLABLE) { return nullptr; } // If an object type T satisfies the appropriate protocol, it is deemed // "arena compatible" and handled more efficiently because this interface // (i) passes the arena pointer to the created object so that its // sub-objects and internal allocations can use the arena too, and (ii) // elides the object's destructor call when possible; e.g. protobuf // messages, RepeatedField, etc. Otherwise, the arena will invoke the // object's destructor when the arena is destroyed. // // To be "arena-compatible", a type T must satisfy the following: // // - The type T must have (at least) two constructors: a constructor // callable with `args` (without `arena`), called when a T is allocated on // the heap; and a constructor callable with `Arena* arena, Args&&... // args`, called when a T is allocated on an arena. If the second // constructor is called with a null arena pointer, it must be equivalent // to invoking the first // (`args`-only) constructor. // // - The type T must have a particular type trait: a nested type // |InternalArenaConstructable_|. This is usually a typedef to |void|. // // - The type T *may* have the type trait |DestructorSkippable_|. If this // type trait is present in the type, then its destructor will not be // called if and only if it was passed a non-null arena pointer. If this // type trait is not present on the type, then its destructor is always // called when the containing arena is destroyed. // // The protocol is implemented by all protobuf message classes as well as // protobuf container types like RepeatedPtrField and Map. It is internal to // protobuf and is not guaranteed to be stable. Non-proto types should not // rely on this protocol. template static char DestructorSkippable( const typename U::DestructorSkippable_* PROTOBUF_NULLABLE); template static double DestructorSkippable(...); typedef std::integral_constant< bool, sizeof(DestructorSkippable(static_cast(nullptr))) == sizeof(char) || std::is_trivially_destructible::value> is_destructor_skippable; template static char ArenaConstructable( const typename U::InternalArenaConstructable_* PROTOBUF_NULLABLE); template static double ArenaConstructable(...); typedef std::integral_constant( static_cast(nullptr))) == sizeof(char)> is_arena_constructable; // Note that by this point, for types `U` which overload `FieldArenaRep`, // `T` is the arena representation `FieldArenaRep::Type` and is expected // to have an arena-enabled constructor. // // For types with a different arena representation, if the arena pointer is // null, the object is allocated directly with `new` as its original type, // since wrapping the type in the arena representation would be wasteful. template static T* PROTOBUF_NONNULL ConstructOnArena(void* PROTOBUF_NONNULL ptr, Arena& arena, Args&&... args) { return new (ptr) T(&arena, static_cast(args)...); } template static T* PROTOBUF_NONNULL Construct(void* PROTOBUF_NONNULL ptr, Arena* PROTOBUF_NULLABLE arena, Args&&... args) { if (ABSL_PREDICT_FALSE(arena == nullptr)) { return new (ptr) T(static_cast(args)...); } else { return ConstructOnArena(ptr, *arena, static_cast(args)...); } } static PROTOBUF_ALWAYS_INLINE T* PROTOBUF_NONNULL New() { // Fields which use arena offsets don't have constructors that take an // arena pointer. Since the arena is nullptr, it is safe to default // construct the object. if constexpr (internal::FieldHasArenaOffset() || internal::HasDeprecatedArenaConstructor()) { return new T(); } else { return new T(nullptr); } } friend class Arena; friend class TestUtil::ReflectionTester; }; // Provides access to protected GetArena to generated messages. // For internal use only. template static Arena* PROTOBUF_NULLABLE InternalGetArena(T* PROTOBUF_NONNULL p) { return InternalHelper::GetArena(p); } // Helper typetraits that indicates support for arenas in a type T at compile // time. This is public only to allow construction of higher-level templated // utilities. // // is_arena_constructable::value is true if the message type T has arena // support enabled, and false otherwise. // // is_destructor_skippable::value is true if the message type T has told // the arena that it is safe to skip the destructor, and false otherwise. // // This is inside Arena because only Arena has the friend relationships // necessary to see the underlying generated code traits. template struct is_arena_constructable : InternalHelper::is_arena_constructable {}; template struct is_destructor_skippable : InternalHelper::is_destructor_skippable { }; private: internal::ThreadSafeArena impl_; enum class ConstructType { kUnknown, kDefault, kCopy, kMove }; // Overload set to detect which kind of construction is going to happen for a // specific set of input arguments. This is used to dispatch to different // helper functions. template static auto ProbeConstructType() -> std::integral_constant; template static auto ProbeConstructType(const T&) -> std::integral_constant; template static auto ProbeConstructType(T&) -> std::integral_constant; template static auto ProbeConstructType(const T&&) -> std::integral_constant; template static auto ProbeConstructType(T&&) -> std::integral_constant; template static auto ProbeConstructType(U&&...) -> std::integral_constant; template static constexpr auto GetConstructType() { return std::is_base_of_v ? decltype(ProbeConstructType(std::declval()...))::value : ConstructType::kUnknown; } void ReturnArrayMemory(void* PROTOBUF_NONNULL p, size_t size) { impl_.ReturnArrayMemory(p, size); } template PROTOBUF_NDEBUG_INLINE static T* PROTOBUF_NONNULL CreateArenaCompatible(Arena* PROTOBUF_NULLABLE arena, Args&&... args) { static_assert(is_arena_constructable::value, "Can only construct types that are ArenaConstructable"); if (ABSL_PREDICT_FALSE(arena == nullptr)) { if constexpr (internal::FieldHasArenaOffset() || internal::HasDeprecatedArenaConstructor()) { return new T(static_cast(args)...); } else { return new T(nullptr, static_cast(args)...); } } else { return arena->DoCreateMessage(static_cast(args)...); } } // This specialization for no arguments is necessary, because its behavior is // slightly different. When the arena pointer is nullptr, it calls T() // instead of T(nullptr). template PROTOBUF_NDEBUG_INLINE static T* PROTOBUF_NONNULL CreateArenaCompatible(Arena* PROTOBUF_NULLABLE arena) { static_assert(is_arena_constructable::value, "Can only construct types that are ArenaConstructable"); if (ABSL_PREDICT_FALSE(arena == nullptr)) { // Generated arena constructor T(Arena*) is protected. Call via // InternalHelper. return InternalHelper::New(); } else { return arena->DoCreateMessage(); } } template > PROTOBUF_NDEBUG_INLINE void* PROTOBUF_NONNULL AllocateInternal() { if (trivial) { return AllocateAligned(sizeof(T), alignof(T)); } else { // We avoid instantiating arena_destruct_object in the trivial case. constexpr auto dtor = &internal::cleanup::arena_destruct_object< std::conditional_t>; return AllocateAlignedWithCleanup(sizeof(T), alignof(T), dtor); } } // DefaultConstruct/CopyConstruct: // // Functions with a generic signature to support taking the address in generic // contexts, like RepeatedPtrField, etc. // These are also used as a hook for `extern template` instantiations where // codegen can offload the instantiations to the respective .pb.cc files. This // has two benefits: // - It reduces the library bloat as callers don't have to instantiate the // function. // - It allows the optimizer to see the constructors called to // further optimize the instantiation. template static void* PROTOBUF_NONNULL DefaultConstruct(Arena* PROTOBUF_NULLABLE arena); template static void* PROTOBUF_NONNULL CopyConstruct( Arena* PROTOBUF_NULLABLE arena, const void* PROTOBUF_NONNULL from); template PROTOBUF_NDEBUG_INLINE T* PROTOBUF_NONNULL DoCreateMessage(Args&&... args) { using ArenaRepT = typename internal::FieldArenaRep::Type; auto* arena_repr = InternalHelper::ConstructOnArena( AllocateInternal::value>(), *this, std::forward(args)...); // Note that we can't static_cast arena_repr to T* here, since T might be a // member of ArenaRepT. return internal::FieldArenaRep::Get(arena_repr); } // CreateInArenaStorage is used to implement map field. Without it, // Map need to call generated message's protected arena constructor, // which needs to declare Map as friend of generated message. template static void CreateInArenaStorage(T* PROTOBUF_NONNULL ptr, Arena* PROTOBUF_NULLABLE arena, Args&&... args) { if constexpr (is_arena_constructable::value) { InternalHelper::Construct(ptr, arena, std::forward(args)...); } else { new (ptr) T(std::forward(args)...); } if constexpr (!is_destructor_skippable::value) { if (ABSL_PREDICT_TRUE(arena != nullptr)) { arena->OwnDestructor(ptr); } } } // Implementation for GetArena(). Only message objects with // InternalArenaConstructable_ tags can be associated with an arena, and such // objects must implement a GetArena() method. template PROTOBUF_ALWAYS_INLINE static Arena* PROTOBUF_NULLABLE GetArenaInternal(T* PROTOBUF_NONNULL value) { return InternalHelper::GetArena(value); } void* PROTOBUF_NONNULL AllocateAlignedForArray(size_t n, size_t align) { if (align <= internal::ArenaAlignDefault::align) { return AllocateForArray(internal::ArenaAlignDefault::Ceil(n)); } else { // We are wasting space by over allocating align - 8 bytes. Compared // to a dedicated function that takes current alignment in consideration. // Such a scheme would only waste (align - 8)/2 bytes on average, but // requires a dedicated function in the outline arena allocation // functions. Possibly re-evaluate tradeoffs later. auto align_as = internal::ArenaAlignAs(align); return align_as.Ceil(AllocateForArray(align_as.Padded(n))); } } void* PROTOBUF_NONNULL Allocate(size_t n); void* PROTOBUF_NONNULL AllocateForArray(size_t n); void* PROTOBUF_NONNULL AllocateAlignedWithCleanup( size_t n, size_t align, void (*PROTOBUF_NONNULL destructor)(void* PROTOBUF_NONNULL)); // Test only API. // It returns the objects that are in the cleanup list for the current // SerialArena. This API is meant for tests that want to see if something was // added or not to the cleanup list. Sometimes adding something to the cleanup // list has no visible side effect so peeking into the list is the only way to // test. std::vector PeekCleanupListForTesting(); template friend class internal::GenericTypeHandler; friend class internal::InternalMetadata; // For user_arena(). friend class internal::LazyField; // For DefaultConstruct. friend class internal::EpsCopyInputStream; // For parser performance friend class internal::TcParser; // For parser performance friend class MessageLite; template friend class Map; template friend class RepeatedField; // For ReturnArrayMemory friend class internal::RepeatedPtrFieldBase; // For ReturnArrayMemory friend class internal::UntypedMapBase; // For ReturnArrayMemory friend class internal::ExtensionSet; // For ReturnArrayMemory friend internal::SerialArena* PROTOBUF_NULLABLE internal::GetSerialArena(Arena* PROTOBUF_NULLABLE); friend struct internal::ArenaTestPeer; }; namespace internal { // Comparison base to inject relational operators in UniquePtr and Ptr. // We use a base class to facilitate symmetric relational operators with // UniquePtr, Ptr, T* and nullptr. struct ArenaPtrCmpBase { template static T* PROTOBUF_NULLABLE Unpack(T* PROTOBUF_NULLABLE ptr) { return ptr; } template static auto PROTOBUF_NULLABLE Unpack(const typename Arena::UniquePtr& ptr) { return ptr.get(); } template static auto PROTOBUF_NONNULL Unpack(const typename Arena::template Ptr& ptr) { return ptr.get(); } static std::nullptr_t Unpack(std::nullptr_t) { return nullptr; } public: template friend auto operator==(const LHS& lhs, const RHS& rhs) -> decltype(Unpack(lhs) == Unpack(rhs)) { return Unpack(lhs) == Unpack(rhs); } template friend auto operator!=(const LHS& lhs, const RHS& rhs) -> decltype(lhs == rhs) { return !(lhs == rhs); } }; // Transparent hasher that supports the same types as equality above. // This allows for heterogeneous lookup on UniquePtr and Ptr keyed associative // containers. struct ArenaPtrContainerHash { using is_transparent = void; template auto operator()(const T& value) const -> decltype(absl::HashOf(ArenaPtrCmpBase::Unpack(value))) { return absl::HashOf(ArenaPtrCmpBase::Unpack(value)); } }; // The deleter type used for implementing UniquePtr. // Only deletes an element if the Arena* passed at construction time is // nullptr. struct UniquePtrDeleter { template void operator()(T* PROTOBUF_NONNULL element) const { if (arena == nullptr) delete element; } Arena* PROTOBUF_NULLABLE arena = nullptr; }; } // namespace internal template class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI ABSL_NULLABILITY_COMPATIBLE PROTOBUF_NULL_AFTER_MOVE Arena::UniquePtr final : internal::ArenaPtrCmpBase { public: using pointer = T*; using element_type = T; // Public Constructors constexpr UniquePtr() : ptr_(nullptr, Deleter{}) {} // NOLINTNEXTLINE(google-explicit-constructor) constexpr UniquePtr(std::nullptr_t) : ptr_(nullptr, Deleter{}) {} // Allow implicit conversion from `std::unique_ptr` with // `std::default_delete`. // This is always safe since `UniquePtr` can safely hold heap-allocated // pointers. // NOLINTNEXTLINE(google-explicit-constructor) UniquePtr(PROTOBUF_NULLABLE std::unique_ptr heap_owned) : ptr_(heap_owned.release(), Deleter{}) {} // Allow implicit conversion from `Ptr`. // This is always safe since `Ptr` is statically known to be owned by an // arena. There is no "unique" ownership on it. // NOLINTNEXTLINE(google-explicit-constructor) UniquePtr(Ptr arena_owned) : UniquePtr(arena_owned.get(), arena_owned.GetOwningArena()) {} ~UniquePtr() = default; constexpr UniquePtr(UniquePtr&& rhs) = default; template >> // NOLINTNEXTLINE(google-explicit-constructor) constexpr UniquePtr(UniquePtr&& rhs) : ptr_(std::move(rhs.ptr_)) {} // Use Arena::UnsafeWrapUniquePtr or Arena::MakeUnique explicit UniquePtr(T* PROTOBUF_NULLABLE ptr) = delete; UniquePtr& operator=(UniquePtr&& rhs) = default; UniquePtr& operator=(std::nullptr_t) { reset(); return *this; } template >> UniquePtr& operator=(UniquePtr&& rhs) { ptr_ = std::move(rhs.ptr_); return *this; } // Delete the copy ctor and copy assignment operator. UniquePtr(const UniquePtr& rhs) = delete; UniquePtr& operator=(const UniquePtr& rhs) = delete; // If heap allocated transfer ownership of the pointer to the caller, clearing // the `UniquePtr` instance. // Otherwise, return `absl::nullopt` and have no effect. absl::optional> try_heap_release() { if (GetOwningArena() != nullptr || get() == nullptr) { return absl::nullopt; } return std::unique_ptr(std::exchange(ptr_, UniquePtrType()).release()); } // If it contains an arena allocated object, return a `Ptr` to the caller. // Otherwise, return `absl::nullopt`. // This function has does not modify the `UniquePtr`. absl::optional> try_as_arena_ptr() const { Arena* arena = GetOwningArena(); if (arena == nullptr || get() == nullptr) { return absl::nullopt; } return Ptr(arena, get()); } void swap(UniquePtr& other) noexcept { ptr_.swap(other.ptr_); } friend void swap(UniquePtr& a, UniquePtr& b) noexcept { a.swap(b); } // reset() the pointed to object to nullptr. ABSL_ATTRIBUTE_REINITIALIZES void reset() { ptr_.reset(); } ABSL_ATTRIBUTE_REINITIALIZES void reset(std::nullptr_t) { ptr_.reset(nullptr); } void reset(T* PROTOBUF_NULLABLE) = delete; PROTOBUF_NULLABLE pointer get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return ptr_.get(); } PROTOBUF_NONNULL pointer operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return get(); } element_type& operator*() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(ptr_ != nullptr); return *ptr_; } explicit operator bool() const { return ptr_ != nullptr; } // Return a pointer to the Arena pointer that owns the pointed to message. Arena* PROTOBUF_NULLABLE GetOwningArena() const { return ptr_.get_deleter().arena; } template friend void AbslStringify(Sink& sink, const UniquePtr& ptr) { if constexpr (std::is_base_of_v) { if (ptr != nullptr) { absl::Format(&sink, "points to (%p) with value <%v>", ptr.get(), *ptr); return; } } absl::Format(&sink, "%p", ptr.get()); } using absl_container_hash = internal::ArenaPtrContainerHash; template friend H AbslHashValue(H h, const UniquePtr& u) { return H::combine(std::move(h), u.ptr_); } private: friend Arena; template friend class ABSL_NULLABILITY_COMPATIBLE UniquePtr; using Deleter = internal::UniquePtrDeleter; using UniquePtrType = std::unique_ptr; // Only allow construction through the helper functions in order to ensure // that the owning_arena passed in did actually come from the parents. UniquePtr(T* PROTOBUF_NULLABLE t, Arena* PROTOBUF_NULLABLE owning_arena) : ptr_(t, Deleter{owning_arena}) {} // The underlying std::unique_ptr member. PROTOBUF_NULLABLE UniquePtrType ptr_; }; template class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI Arena::Ptr final : internal::ArenaPtrCmpBase { public: using pointer = T*; using element_type = T; constexpr Ptr(const Ptr& rhs) = default; template >> // NOLINTNEXTLINE(google-explicit-constructor) constexpr Ptr(const Ptr& rhs) : ptr_(rhs.ptr_), arena_(rhs.arena_) {} ~Ptr() = default; Ptr& operator=(const Ptr& rhs) = default; template >> Ptr& operator=(const Ptr& rhs) { ptr_ = rhs.ptr_; arena_ = rhs.arena_; return *this; } void swap(Ptr& other) noexcept { std::swap(ptr_, other.ptr_); std::swap(arena_, other.arena_); } friend void swap(Ptr& a, Ptr& b) noexcept { a.swap(b); } PROTOBUF_NONNULL pointer get() const { return ptr_; } PROTOBUF_NONNULL pointer operator->() const { return ptr_; } element_type& operator*() const { return *ptr_; } // Return a pointer to the Arena pointer that owns the pointed to message. Arena* PROTOBUF_NONNULL GetOwningArena() const { return arena_; } template friend void AbslStringify(Sink& sink, Ptr ptr) { if constexpr (std::is_base_of_v) { absl::Format(&sink, "points to (%p) with value <%v>", ptr.get(), *ptr); } else { absl::Format(&sink, "%p", ptr.get()); } } using absl_container_hash = internal::ArenaPtrContainerHash; template friend H AbslHashValue(H h, Ptr u) { return H::combine(std::move(h), u.ptr_); } private: friend Arena; template friend class Ptr; // Only allow construction through the helper functions in order to ensure // that the owning_arena passed in did actually come from the parents. Ptr(Arena* PROTOBUF_NONNULL owning_arena, T* PROTOBUF_NONNULL ptr) : ptr_(ptr), arena_(owning_arena) {} T* PROTOBUF_NONNULL ptr_; Arena* PROTOBUF_NONNULL arena_; }; // DefaultConstruct/CopyConstruct // // IMPORTANT: These have to be defined out of line and without an `inline` // keyword to make sure the `extern template` suppresses instantiations. template PROTOBUF_NOINLINE void* PROTOBUF_NONNULL Arena::DefaultConstruct(Arena* PROTOBUF_NULLABLE arena) { if constexpr (internal::FieldHasArenaOffset()) { if (arena != nullptr) { using ArenaRepT = typename internal::FieldArenaRep::Type; static_assert(is_destructor_skippable::value); void* mem = arena->AllocateAligned(sizeof(ArenaRepT)); ArenaRepT* arena_rep = new (mem) ArenaRepT(arena); return internal::FieldArenaRep::Get(arena_rep); } else { static_assert(is_destructor_skippable::value); // Fields which use arena offsets don't have constructors that take an // arena pointer. Since the arena is nullptr, it is safe to default // construct the object. return new (internal::Allocate(sizeof(T))) T(); } } else { static_assert(is_destructor_skippable::value); void* mem = arena != nullptr ? arena->AllocateAligned(sizeof(T)) : internal::Allocate(sizeof(T)); if constexpr (internal::HasDeprecatedArenaConstructor()) { return new (mem) T(internal::InternalVisibility(), arena); } else { return new (mem) T(arena); } } } template PROTOBUF_NOINLINE void* PROTOBUF_NONNULL Arena::CopyConstruct( Arena* PROTOBUF_NULLABLE arena, const void* PROTOBUF_NONNULL from) { const auto* typed_from = static_cast(from); // If the object is larger than half of a cache line, prefetch either the rest // of it or half of it, whichiver is smaller, starting at 1-cache-line offset. // This has shown the best benchmark results on average between several tested // configurations. if constexpr (sizeof(T) > ABSL_CACHELINE_SIZE / 2) { using internal::PrefetchOpts; static constexpr PrefetchOpts kPrefetchOpts = { /*num=*/{std::min(sizeof(T) / 2, sizeof(T) - ABSL_CACHELINE_SIZE / 2), PrefetchOpts::kBytes}, /*from=*/{1, PrefetchOpts::kLines}, /*locality=*/PrefetchOpts::kHigh, }; internal::Prefetch(typed_from); } static_assert(is_destructor_skippable::value, ""); void* mem; if (arena != nullptr) { mem = arena->AllocateAligned(sizeof(T)); } else { mem = internal::Allocate(sizeof(T)); } return new (mem) T(arena, *typed_from); } template <> inline void* PROTOBUF_NONNULL Arena::AllocateInternal() { return impl_.AllocateFromStringBlock(); } namespace internal { inline SerialArena* PROTOBUF_NULLABLE GetSerialArena(SerialArena* PROTOBUF_NULLABLE arena) { return arena; } inline SerialArena* PROTOBUF_NULLABLE GetSerialArena(Arena* PROTOBUF_NULLABLE arena) { if (arena == nullptr) return nullptr; SerialArena* res = arena->impl_.GetSerialArena(); PROTOBUF_ASSUME(res != nullptr); return res; } // Using a template to make member access type dependent and delay it until // instantiation when `MessageLite` will be complete. // Not really a generic function. template inline SerialArena* PROTOBUF_NULLABLE GetSerialArena(const MessageLite* PROTOBUF_NONNULL elem) { const auto* dependent_elem = (delay, ..., elem); return GetSerialArena(dependent_elem->GetArena()); } // This class is used to define `DestructorSkippable_` for some containing type // if and only if `T` is destructor-skippable. template ::value> struct ContainerDestructorSkippableBase {}; template struct ContainerDestructorSkippableBase { using DestructorSkippable_ = void; }; } // namespace internal } // namespace protobuf } // namespace google #include "google/protobuf/port_undef.inc" #endif // GOOGLE_PROTOBUF_ARENA_H__