Cold-Air-Networks/mirror-llvm-project
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 7

[libc] Migrate Block to BlockRef in baremetal allocator (#201001)

Browse source 
Under C++ object lifetime and strict aliasing rules, accessing typed
objects requires that an object of that type actually exists at the
memory location. Previously, the Block structure stored prev and next
offset values, where the prev field overlapped with the usable space of
the preceding block to save space. When the predecessor was allocated,
user payload was written directly to this overlapping space,
complicating object lifetime management. A key issue arose during
reallocation (like in-place shrinking), where the allocator needed to
manipulate block boundaries (e.g., splitting a block) while user payload
was still actively residing in that memory. This caused undefined
behavior due to accessing typed Block members that collided with the
user's active objects.

This patch eliminates the problem by always treating blocks as raw bytes
accessed through a byte-backed proxy (BlockRef). Instead of constructing
or casting to typed Block structures, metadata is read and written using
aligned byte-copy operations (inline_memcpy). This decouples block
manipulation from C++ object lifetime rules, guaranteeing safe access
without violating strict aliasing.

Assisted-by: AI Tools, checked manually
This commit is contained in:
Schrodinger ZHU Yifan 2026-06-09 16:33:04 -04:00 committed by GitHub
parent 596e95cc51
commit 003ba404c8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 651 additions and 542 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
libc/fuzzing/__support/freelist_heap_fuzz.cpp
View file

@ -1,4 +1,4 @@
//===-- freelist_heap_fuzz.cpp --------------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Fuzzing test for llvm-libc freelist-based heap implementation.
///
//===----------------------------------------------------------------------===//
@ -26,7 +27,7 @@ _end:
__llvm_libc_heap_limit:
)");
using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::inline_memset;
using LIBC_NAMESPACE::cpp::nullopt;
@ -148,7 +149,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
// Perform allocation.
void *ptr = nullptr;
size_t alignment = Block::MIN_ALIGN;
size_t alignment = BlockRef::MIN_ALIGN;
switch (alloc_type) {
case AllocType::MALLOC:
ptr = heap.allocate(alloc_size);
@ -173,7 +174,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
alloc_size - alloc.size);
alloc.ptr = ptr;
alloc.size = alloc_size;
alloc.alignment = Block::MIN_ALIGN;
alloc.alignment = BlockRef::MIN_ALIGN;
}
break;
}
@ -195,8 +196,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
if (ptr) {
// aligned_allocate should automatically apply a minimum alignment.
if (alignment < Block::MIN_ALIGN)
alignment = Block::MIN_ALIGN;
if (alignment < BlockRef::MIN_ALIGN)
alignment = BlockRef::MIN_ALIGN;
// Check alignment.
if (reinterpret_cast<uintptr_t>(ptr) % alignment)
__builtin_trap();

341
libc/src/__support/block.h
View file

@ -1,25 +1,31 @@
//===-- Implementation header for a block of memory -------------*- C++ -*-===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Implementation header for a block of memory.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_BLOCK_H
#define LLVM_LIBC_SRC___SUPPORT_BLOCK_H
#include "hdr/stdint_proxy.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/algorithm.h"
#include "src/__support/CPP/cstddef.h"
#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/new.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/CPP/type_traits.h"
#include "src/__support/libc_assert.h"
#include "src/__support/macros/config.h"
#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
namespace LIBC_NAMESPACE_DECL {
@ -38,15 +44,17 @@ LIBC_INLINE constexpr size_t align_up(size_t value, size_t alignment) {
using ByteSpan = cpp::span<LIBC_NAMESPACE::cpp::byte>;
using cpp::optional;
/// Memory region with links to adjacent blocks.
/// Reference to a memory region with links to adjacent blocks.
///
/// The blocks store their offsets to the previous and next blocks. The latter
/// is also the block's size.
/// is also the block's size. The metadata is stored in raw bytes and accessed
/// through aligned byte-copy loads and stores so the header can overlap user
/// storage without creating typed aliasing accesses.
///
/// All blocks have their usable space aligned to some multiple of MIN_ALIGN.
/// This also implies that block outer sizes are aligned to MIN_ALIGN.
///
/// As an example, the diagram below represents two contiguous `Block`s. The
/// As an example, the diagram below represents two contiguous blocks. The
/// indices indicate byte offsets:
///
/// @code{.unparsed}
@ -90,48 +98,71 @@ using cpp::optional;
///
/// The next offset of a block matches the previous offset of its next block.
/// The first block in a list is denoted by having a previous offset of `0`.
class Block {
// Masks for the contents of the next_ field.
class BlockRef {
// Masks for the contents of the next field.
static constexpr size_t PREV_FREE_MASK = 1 << 0;
static constexpr size_t LAST_MASK = 1 << 1;
static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK);
// Header field offsets. The value at PREV_OFFSET is only meaningful when the
// PREV_FREE_MASK bit is set in the next field.
static constexpr size_t PREV_OFFSET = 0;
static constexpr size_t NEXT_OFFSET = PREV_OFFSET + sizeof(size_t);
public:
static constexpr size_t HEADER_SIZE = NEXT_OFFSET + sizeof(size_t);
// To ensure block sizes have two lower unused bits, ensure usable space is
// always aligned to at least 4 bytes. (The distances between usable spaces,
// the outer size, is then always also 4-aligned.)
static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t));
// No copy or move.
Block(const Block &other) = delete;
Block &operator=(const Block &other) = delete;
LIBC_INLINE constexpr BlockRef() = default;
LIBC_INLINE explicit constexpr BlockRef(cpp::byte *header_ptr)
: header_ptr(header_ptr) {}
LIBC_INLINE explicit constexpr operator bool() const {
return header_ptr != nullptr;
}
LIBC_INLINE constexpr bool operator==(BlockRef other) const {
return header_ptr == other.header_ptr;
}
LIBC_INLINE constexpr bool operator!=(BlockRef other) const {
return !(*this == other);
}
LIBC_INLINE uintptr_t addr() const {
return reinterpret_cast<uintptr_t>(header_ptr);
}
/// Initializes a given memory region into a first block and a sentinel last
/// block. Returns the first block, which has its usable space aligned to
/// MIN_ALIGN.
static optional<Block *> init(ByteSpan region);
static optional<BlockRef> init(ByteSpan region);
/// @returns A pointer to a `Block`, given a pointer to the start of the
/// usable space inside the block.
/// @returns A pointer to a block, given a pointer to the start of the usable
/// space inside the block.
///
/// This is the inverse of `usable_space()`.
///
/// @warning This method does not do any checking; passing a random
/// pointer will return a non-null pointer.
LIBC_INLINE static Block *from_usable_space(void *usable_space) {
/// @warning This method does not do any checking; passing a random pointer
/// will return a non-null pointer.
LIBC_INLINE static BlockRef from_usable_space(void *usable_space) {
LIBC_ASSERT(usable_space != nullptr && "usable space cannot be null");
auto *bytes = reinterpret_cast<cpp::byte *>(usable_space);
return reinterpret_cast<Block *>(bytes - sizeof(Block));
return BlockRef(bytes - HEADER_SIZE);
}
LIBC_INLINE static const Block *from_usable_space(const void *usable_space) {
LIBC_INLINE static BlockRef from_usable_space(const void *usable_space) {
LIBC_ASSERT(usable_space != nullptr && "usable space cannot be null");
const auto *bytes = reinterpret_cast<const cpp::byte *>(usable_space);
return reinterpret_cast<const Block *>(bytes - sizeof(Block));
return BlockRef(const_cast<cpp::byte *>(bytes - HEADER_SIZE));
}
/// @returns The total size of the block in bytes, including the header.
LIBC_INLINE size_t outer_size() const { return next_ & SIZE_MASK; }
LIBC_INLINE size_t outer_size() const { return load_next() & SIZE_MASK; }
LIBC_INLINE static size_t outer_size(size_t inner_size) {
// The usable region includes the prev_ field of the next block.
return inner_size - sizeof(prev_) + sizeof(Block);
LIBC_INLINE static constexpr size_t outer_size(size_t inner_size) {
// The usable region includes the prev field of the next block.
return inner_size - PREV_FIELD_SIZE + HEADER_SIZE;
}
/// @returns The number of usable bytes inside the block were it to be
@ -144,9 +175,9 @@ public:
/// @returns The number of usable bytes inside a block with the given outer
/// size were it to be allocated.
LIBC_INLINE static size_t inner_size(size_t outer_size) {
// The usable region includes the prev_ field of the next block.
return inner_size_free(outer_size) + sizeof(prev_);
LIBC_INLINE static constexpr size_t inner_size(size_t outer_size) {
// The usable region includes the prev field of the next block.
return inner_size_free(outer_size) + PREV_FIELD_SIZE;
}
/// @returns The number of usable bytes inside the block if it remains free.
@ -158,87 +189,70 @@ public:
/// @returns The number of usable bytes inside a block with the given outer
/// size if it remains free.
LIBC_INLINE static size_t inner_size_free(size_t outer_size) {
return outer_size - sizeof(Block);
LIBC_INLINE static constexpr size_t inner_size_free(size_t outer_size) {
return outer_size - HEADER_SIZE;
}
/// @returns A pointer to the usable space inside this block.
///
/// Aligned to some multiple of MIN_ALIGN.
LIBC_INLINE cpp::byte *usable_space() {
auto *s = reinterpret_cast<cpp::byte *>(this) + sizeof(Block);
LIBC_ASSERT(reinterpret_cast<uintptr_t>(s) % MIN_ALIGN == 0 &&
"usable space must be aligned to MIN_ALIGN");
return s;
}
LIBC_INLINE const cpp::byte *usable_space() const {
const auto *s = reinterpret_cast<const cpp::byte *>(this) + sizeof(Block);
LIBC_INLINE cpp::byte *usable_space() const {
auto *s = nonnull_header_ptr() + HEADER_SIZE;
LIBC_ASSERT(reinterpret_cast<uintptr_t>(s) % MIN_ALIGN == 0 &&
"usable space must be aligned to MIN_ALIGN");
return s;
}
// @returns The region of memory the block manages, including the header.
LIBC_INLINE ByteSpan region() {
return {reinterpret_cast<cpp::byte *>(this), outer_size()};
LIBC_INLINE ByteSpan region() const {
return {nonnull_header_ptr(), outer_size()};
}
/// Attempts to split this block.
///
/// If successful, the block will have an inner size of at least
/// `new_inner_size`. The remaining space will be returned as a new block,
/// with usable space aligned to `usable_space_alignment`. Note that the prev_
/// with usable space aligned to `usable_space_alignment`. Note that the prev
/// field of the next block counts as part of the inner size of the block.
/// `usable_space_alignment` must be a multiple of MIN_ALIGN.
optional<Block *> split(size_t new_inner_size,
size_t usable_space_alignment = MIN_ALIGN);
optional<BlockRef> split(size_t new_inner_size,
size_t usable_space_alignment = MIN_ALIGN) const;
/// Merges this block with the one that comes after it.
bool merge_next();
bool merge_next() const;
/// @returns The block immediately after this one, or a null pointer if this
/// is the last block.
LIBC_INLINE Block *next() const {
if (next_ & LAST_MASK)
return nullptr;
return reinterpret_cast<Block *>(reinterpret_cast<uintptr_t>(this) +
outer_size());
/// @returns The block immediately after this one, or a null block if this is
/// the last block.
LIBC_INLINE BlockRef next() const {
size_t next_value = load_next();
if (next_value & LAST_MASK)
return BlockRef();
return BlockRef(nonnull_header_ptr() + (next_value & SIZE_MASK));
}
/// @returns The free block immediately before this one, otherwise nullptr.
LIBC_INLINE Block *prev_free() const {
if (!(next_ & PREV_FREE_MASK))
return nullptr;
return reinterpret_cast<Block *>(reinterpret_cast<uintptr_t>(this) - prev_);
/// @returns The free block immediately before this one, otherwise null.
LIBC_INLINE BlockRef prev_free() const {
if (!(load_next() & PREV_FREE_MASK))
return BlockRef();
return BlockRef(nonnull_header_ptr() - load_prev());
}
/// @returns Whether the block is unavailable for allocation.
LIBC_INLINE bool used() const { return !next() || !next()->prev_free(); }
LIBC_INLINE bool used() const { return !next() || !next().prev_free(); }
/// Marks this block as in use.
LIBC_INLINE void mark_used() {
LIBC_INLINE void mark_used() const {
LIBC_ASSERT(next() && "last block is always considered used");
next()->next_ &= ~PREV_FREE_MASK;
BlockRef next_block = next();
next_block.store_next(next_block.load_next() & ~PREV_FREE_MASK);
}
/// Marks this block as free.
LIBC_INLINE void mark_free() {
LIBC_INLINE void mark_free() const {
LIBC_ASSERT(next() && "last block is always considered used");
next()->next_ |= PREV_FREE_MASK;
// The next block's prev_ field becomes alive, as it is no longer part of
// this block's used space.
*new (&next()->prev_) size_t = outer_size();
}
LIBC_INLINE Block(size_t outer_size, bool is_last) : next_(outer_size) {
// Last blocks are not usable, so they need not have sizes aligned to
// MIN_ALIGN.
LIBC_ASSERT(outer_size % (is_last ? alignof(Block) : MIN_ALIGN) == 0 &&
"block sizes must be aligned");
LIBC_ASSERT(is_usable_space_aligned(MIN_ALIGN) &&
"usable space must be aligned to a multiple of MIN_ALIGN");
if (is_last)
next_ |= LAST_MASK;
BlockRef next_block = next();
next_block.store_next(next_block.load_next() | PREV_FREE_MASK);
next_block.store_prev(outer_size());
}
LIBC_INLINE bool is_usable_space_aligned(size_t alignment) const {
@ -259,7 +273,7 @@ public:
// We must create a new block inside this one (splitting). This requires a
// block header in addition to the requested size.
if (add_overflow(size, sizeof(Block), size))
if (add_overflow(size, HEADER_SIZE, size))
return 0;
// Beyond that, padding space may need to remain in this block to ensure
@ -275,79 +289,81 @@ public:
// So the maximum distance would be G - L. As a special case, if L is 1
// (unaligned), the max distance is G - 1.
//
// This block's usable space is aligned to MIN_ALIGN >= Block. With zero
// padding, the next block's usable space is sizeof(Block) past it, which is
// a point aligned to Block. Thus the max padding needed is alignment -
// alignof(Block).
if (add_overflow(size, alignment - alignof(Block), size))
// This block's usable space is aligned to MIN_ALIGN >= header alignment.
// With zero padding, the next block's usable space is HEADER_SIZE past it,
// which is aligned to header alignment. Thus the max padding needed is
// alignment - alignof(size_t).
if (add_overflow(size, alignment - alignof(size_t), size))
return 0;
return size;
}
// This is the return type for `allocate` which can split one block into up to
// three blocks.
struct BlockInfo {
// This is the newly aligned block. It will have the alignment requested by
// a call to `allocate` and at most `size`.
Block *block;
// If the usable_space in the new block was not aligned according to the
// `alignment` parameter, we will need to split into this block and the
// `block` to ensure `block` is properly aligned. In this case, `prev` will
// be a pointer to this new "padding" block. `prev` will be nullptr if no
// new block was created or we were able to merge the block before the
// original block with the "padding" block.
Block *prev;
// This is the remainder of the next block after splitting the `block`
// according to `size`. This can happen if there's enough space after the
// `block`.
Block *next;
};
struct BlockInfo;
// Divide a block into up to 3 blocks according to `BlockInfo`. Behavior is
// undefined if allocation is not possible for the given size and alignment.
static BlockInfo allocate(Block *block, size_t alignment, size_t size);
static BlockInfo allocate(BlockRef block, size_t alignment, size_t size);
// These two functions may wrap around.
LIBC_INLINE static uintptr_t
next_possible_block_start(uintptr_t ptr,
size_t usable_space_alignment = MIN_ALIGN) {
return align_up(ptr + sizeof(Block), usable_space_alignment) -
sizeof(Block);
return align_up(ptr + HEADER_SIZE, usable_space_alignment) - HEADER_SIZE;
}
LIBC_INLINE static uintptr_t
prev_possible_block_start(uintptr_t ptr,
size_t usable_space_alignment = MIN_ALIGN) {
return align_down(ptr, usable_space_alignment) - sizeof(Block);
return align_down(ptr, usable_space_alignment) - HEADER_SIZE;
}
/// Only for testing.
static constexpr size_t PREV_FIELD_SIZE = sizeof(size_t);
private:
/// Construct a block to represent a span of bytes. Overwrites only enough
/// memory for the block header; the rest of the span is left alone.
LIBC_INLINE static Block *as_block(ByteSpan bytes) {
LIBC_ASSERT(reinterpret_cast<uintptr_t>(bytes.data()) % alignof(Block) ==
LIBC_INLINE static BlockRef as_block(ByteSpan bytes) {
LIBC_ASSERT(reinterpret_cast<uintptr_t>(bytes.data()) % alignof(size_t) ==
0 &&
"block start must be suitably aligned");
return ::new (bytes.data()) Block(bytes.size(), /*is_last=*/false);
LIBC_ASSERT(bytes.size() % MIN_ALIGN == 0 &&
"block size must be aligned to MIN_ALIGN");
BlockRef block(bytes.data());
block.store_next(bytes.size());
return block;
}
LIBC_INLINE static void make_last_block(cpp::byte *start) {
LIBC_ASSERT(reinterpret_cast<uintptr_t>(start) % alignof(Block) == 0 &&
LIBC_ASSERT(reinterpret_cast<uintptr_t>(start) % alignof(size_t) == 0 &&
"block start must be suitably aligned");
::new (start) Block(sizeof(Block), /*is_last=*/true);
BlockRef last(start);
last.store_next(HEADER_SIZE | LAST_MASK);
}
LIBC_INLINE cpp::byte *field_ptr(size_t offset) const {
cpp::byte *ptr = nonnull_header_ptr() + offset;
LIBC_ASSERT(reinterpret_cast<uintptr_t>(ptr) % alignof(size_t) == 0 &&
"block metadata fields must be aligned");
return ptr;
}
LIBC_INLINE size_t load_field(size_t offset) const {
return *reinterpret_cast<const size_t *>(field_ptr(offset));
}
LIBC_INLINE void store_field(size_t offset, size_t value) const {
new (field_ptr(offset)) size_t(value);
}
/// Offset from this block to the previous block. 0 if this is the first
/// block. This field is only alive when the previous block is free;
/// otherwise, its memory is reused as part of the previous block's usable
/// space.
size_t prev_ = 0;
LIBC_INLINE size_t load_prev() const { return load_field(PREV_OFFSET); }
/// Offset from this block to the next block. Valid even if this is the last
/// block, since it equals the size of the block.
size_t next_ = 0;
///
/// Information about the current state of the block is stored in the two low
/// order bits of the next_ value. These are guaranteed free by a minimum
/// alignment (and thus, alignment of the size) of 4. The lowest bit is the
@ -357,14 +373,49 @@ private:
/// previous block is free.
/// * If the `last` flag is set, the block is the sentinel last block. It is
/// summarily considered used and has no next block.
LIBC_INLINE size_t load_next() const { return load_field(NEXT_OFFSET); }
public:
/// Only for testing.
static constexpr size_t PREV_FIELD_SIZE = sizeof(prev_);
LIBC_INLINE void store_prev(size_t value) const {
store_field(PREV_OFFSET, value);
}
LIBC_INLINE void store_next(size_t value) const {
store_field(NEXT_OFFSET, value);
}
LIBC_INLINE cpp::byte *nonnull_header_ptr() const {
LIBC_ASSERT(header_ptr != nullptr && "operation on an invalid block");
#if __has_builtin(__builtin_assume)
__builtin_assume(header_ptr != nullptr);
#endif
return header_ptr;
}
cpp::byte *header_ptr = nullptr;
};
// This is the return type for `allocate` which can split one block into up to
// three blocks.
struct BlockRef::BlockInfo {
// This is the newly aligned block. It will have the alignment requested by a
// call to `allocate` and at most `size`.
BlockRef block;
// If the usable_space in the new block was not aligned according to the
// `alignment` parameter, we will need to split into this block and the
// `block` to ensure `block` is properly aligned. In this case, `prev` will be
// this new "padding" block. `prev` will be null if no new block was created
// or we were able to merge the block before the original block with the
// "padding" block.
BlockRef prev;
// This is the remainder of the next block after splitting the `block`
// according to `size`. This can happen if there's enough space after the
// `block`.
BlockRef next;
};
LIBC_INLINE
optional<Block *> Block::init(ByteSpan region) {
optional<BlockRef> BlockRef::init(ByteSpan region) {
if (!region.data())
return {};
@ -381,63 +432,66 @@ optional<Block *> Block::init(ByteSpan region) {
if (last_start >= end)
return {};
if (block_start + sizeof(Block) > last_start)
if (block_start + HEADER_SIZE > last_start)
return {};
auto *last_start_ptr = reinterpret_cast<cpp::byte *>(last_start);
Block *block =
BlockRef block =
as_block({reinterpret_cast<cpp::byte *>(block_start), last_start_ptr});
make_last_block(last_start_ptr);
block->mark_free();
block.mark_free();
return block;
}
LIBC_INLINE
Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) {
BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
size_t size) {
LIBC_ASSERT(block.header_ptr != nullptr &&
"allocating from an invalid block");
LIBC_ASSERT(alignment % MIN_ALIGN == 0 &&
"alignment must be a multiple of MIN_ALIGN");
BlockInfo info{block, /*prev=*/nullptr, /*next=*/nullptr};
BlockInfo info{block, BlockRef(), BlockRef()};
if (!info.block->is_usable_space_aligned(alignment)) {
Block *original = info.block;
if (!info.block.is_usable_space_aligned(alignment)) {
BlockRef original = info.block;
// The padding block has no minimum size requirement.
optional<Block *> maybe_aligned_block = original->split(0, alignment);
optional<BlockRef> maybe_aligned_block = original.split(0, alignment);
LIBC_ASSERT(maybe_aligned_block.has_value() &&
"it should always be possible to split for alignment");
if (Block *prev = original->prev_free()) {
if (BlockRef prev = original.prev_free()) {
// If there is a free block before this, we can merge the current one with
// the newly created one.
prev->merge_next();
prev.merge_next();
} else {
info.prev = original;
}
Block *aligned_block = *maybe_aligned_block;
LIBC_ASSERT(aligned_block->is_usable_space_aligned(alignment) &&
BlockRef aligned_block = *maybe_aligned_block;
LIBC_ASSERT(aligned_block.is_usable_space_aligned(alignment) &&
"The aligned block isn't aligned somehow.");
info.block = aligned_block;
}
// Now get a block for the requested size.
if (optional<Block *> next = info.block->split(size))
if (optional<BlockRef> next = info.block.split(size))
info.next = *next;
return info;
}
LIBC_INLINE
optional<Block *> Block::split(size_t new_inner_size,
size_t usable_space_alignment) {
optional<BlockRef> BlockRef::split(size_t new_inner_size,
size_t usable_space_alignment) const {
LIBC_ASSERT(usable_space_alignment % MIN_ALIGN == 0 &&
"alignment must be a multiple of MIN_ALIGN");
// Compute the minimum outer size that produces a block of at least
// `new_inner_size`.
size_t min_outer_size = outer_size(cpp::max(new_inner_size, sizeof(prev_)));
size_t min_outer_size = outer_size(cpp::max(new_inner_size, PREV_FIELD_SIZE));
uintptr_t start = reinterpret_cast<uintptr_t>(this);
uintptr_t start = addr();
uintptr_t next_block_start =
next_possible_block_start(start + min_outer_size, usable_space_alignment);
if (next_block_start < start)
@ -447,33 +501,32 @@ optional<Block *> Block::split(size_t new_inner_size,
"new size must be aligned to MIN_ALIGN");
if (outer_size() < new_outer_size ||
outer_size() - new_outer_size < sizeof(Block))
outer_size() - new_outer_size < HEADER_SIZE)
return {};
bool was_free = !used();
ByteSpan new_region = region().subspan(new_outer_size);
next_ &= ~SIZE_MASK;
next_ |= new_outer_size;
store_next((load_next() & ~SIZE_MASK) | new_outer_size);
Block *new_block = as_block(new_region);
new_block->mark_free();
BlockRef new_block = as_block(new_region);
new_block.mark_free();
if (was_free)
mark_free();
LIBC_ASSERT(new_block->is_usable_space_aligned(usable_space_alignment) &&
LIBC_ASSERT(new_block.is_usable_space_aligned(usable_space_alignment) &&
"usable space must have requested alignment");
return new_block;
}
LIBC_INLINE
bool Block::merge_next() {
if (used() || next()->used())
bool BlockRef::merge_next() const {
BlockRef next_block = next();
if (used() || next_block.used())
return false;
size_t new_size = outer_size() + next()->outer_size();
next_ &= ~SIZE_MASK;
next_ |= new_size;
next()->prev_ = new_size;
size_t new_size = outer_size() + next_block.outer_size();
store_next((load_next() & ~SIZE_MASK) | new_size);
next().store_prev(new_size);
return true;
}

11
libc/src/__support/freelist.cpp
View file

@ -1,10 +1,15 @@
//===-- Implementation for freelist ---------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Implementation for freelist.
///
//===----------------------------------------------------------------------===//
#include "freelist.h"
@ -12,8 +17,8 @@ namespace LIBC_NAMESPACE_DECL {
void FreeList::push(Node *node) {
if (begin_) {
LIBC_ASSERT(Block::from_usable_space(node)->outer_size() ==
begin_->block()->outer_size() &&
LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
begin_->block().outer_size() &&
"freelist entries must have the same size");
// Since the list is circular, insert the node immediately before begin_.
node->prev = begin_->prev;

25
libc/src/__support/freelist.h
View file

@ -1,10 +1,15 @@
//===-- Interface for freelist --------------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Interface for freelist.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
@ -26,15 +31,15 @@ public:
class Node {
public:
/// @returns The block containing this node.
LIBC_INLINE const Block *block() const {
return Block::from_usable_space(this);
LIBC_INLINE BlockRef block() const {
return BlockRef::from_usable_space(this);
}
/// @returns The block containing this node.
LIBC_INLINE Block *block() { return Block::from_usable_space(this); }
LIBC_INLINE BlockRef block() { return BlockRef::from_usable_space(this); }
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block()->inner_size(); }
LIBC_INLINE size_t size() const { return block().inner_size(); }
private:
// Circularly linked pointers to adjacent nodes.
@ -58,16 +63,16 @@ public:
LIBC_INLINE Node *begin() { return begin_; }
/// @returns The first block in the list.
LIBC_INLINE Block *front() { return begin_->block(); }
LIBC_INLINE BlockRef front() { return begin_->block(); }
/// Push a block to the back of the list.
/// The block must be large enough to contain a node.
LIBC_INLINE void push(Block *block) {
LIBC_ASSERT(!block->used() &&
LIBC_INLINE void push(BlockRef block) {
LIBC_ASSERT(!block.used() &&
"only free blocks can be placed on free lists");
LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList) &&
LIBC_ASSERT(block.inner_size_free() >= sizeof(FreeList) &&
"block too small to accomodate free list node");
push(new (block->usable_space()) Node);
push(new (block.usable_space()) Node);
}
/// Push an already-constructed node to the back of the list.

100
libc/src/__support/freelist_heap.h
View file

@ -1,10 +1,15 @@
//===-- Interface for freelist_heap ---------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Interface for freelist_heap.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
@ -53,12 +58,12 @@ private:
void *allocate_impl(size_t alignment, size_t size);
bool shrink_in_place(Block *block, size_t size);
span<cpp::byte> block_to_span(Block *block) {
return span<cpp::byte>(block->usable_space(), block->inner_size());
span<cpp::byte> block_to_span(BlockRef block) {
return span<cpp::byte>(block.usable_space(), block.inner_size());
}
bool shrink_in_place(BlockRef block, size_t size);
bool is_valid_ptr(void *ptr) { return ptr >= begin && ptr < end; }
cpp::byte *begin;
@ -77,9 +82,9 @@ private:
LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = Block::init(region());
Block *block = *result;
free_store.set_range({0, cpp::bit_ceil(block->inner_size())});
auto result = BlockRef::init(region());
BlockRef block = *result;
free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
@ -91,26 +96,26 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!is_initialized)
init();
size_t request_size = Block::min_size_for_allocation(alignment, size);
size_t request_size = BlockRef::min_size_for_allocation(alignment, size);
if (!request_size)
return nullptr;
Block *block = free_store.remove_best_fit(request_size);
BlockRef block = free_store.remove_best_fit(request_size);
if (!block)
return nullptr;
auto block_info = Block::allocate(block, alignment, size);
auto block_info = BlockRef::allocate(block, alignment, size);
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
free_store.insert(block_info.prev);
block_info.block->mark_used();
return block_info.block->usable_space();
block_info.block.mark_used();
return block_info.block.usable_space();
}
LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
return allocate_impl(Block::MIN_ALIGN, size);
return allocate_impl(BlockRef::MIN_ALIGN, size);
}
LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
@ -123,8 +128,8 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
if (size % alignment != 0)
return nullptr;
// The minimum alignment supported by Block is MIN_ALIGN.
alignment = cpp::max(alignment, Block::MIN_ALIGN);
// The minimum alignment supported by BlockRef is MIN_ALIGN.
alignment = cpp::max(alignment, BlockRef::MIN_ALIGN);
return allocate_impl(alignment, size);
}
@ -137,53 +142,48 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
LIBC_ASSERT(is_valid_ptr(bytes) && "Invalid pointer");
Block *block = Block::from_usable_space(bytes);
LIBC_ASSERT(block->next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block->used() && "double free");
block->mark_free();
BlockRef block = BlockRef::from_usable_space(bytes);
LIBC_ASSERT(block.next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block.used() && "double free");
block.mark_free();
// Can we combine with the left or right blocks?
Block *prev_free = block->prev_free();
Block *next = block->next();
BlockRef prev_free = block.prev_free();
BlockRef next = block.next();
if (prev_free != nullptr) {
if (prev_free) {
// Remove from free store and merge.
free_store.remove(prev_free);
block = prev_free;
block->merge_next();
block.merge_next();
}
if (!next->used()) {
if (!next.used()) {
free_store.remove(next);
block->merge_next();
block.merge_next();
}
// Add back to the freelist
free_store.insert(block);
}
LIBC_INLINE bool FreeListHeap::shrink_in_place(Block *block, size_t size) {
size_t min_outer_size = Block::outer_size(cpp::max(size, sizeof(size_t)));
uintptr_t next_block_start = Block::next_possible_block_start(
reinterpret_cast<uintptr_t>(block) + min_outer_size, Block::MIN_ALIGN);
size_t new_outer_size = next_block_start - reinterpret_cast<uintptr_t>(block);
if (block->outer_size() >= new_outer_size) {
// TODO: this is a temporary workaround due to Block's setting prev_
// in its constructor. This code should be deleted once we finishing
// refactoring.
cpp::byte *overlap_ptr =
reinterpret_cast<cpp::byte *>(block) + new_outer_size;
size_t backup;
LIBC_NAMESPACE::inline_memcpy(&backup, overlap_ptr, sizeof(size_t));
optional<Block *> next = block->split(size);
LIBC_NAMESPACE::inline_memcpy(overlap_ptr, &backup, sizeof(size_t));
LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
size_t min_outer_size = BlockRef::outer_size(cpp::max(size, sizeof(size_t)));
uintptr_t next_block_start = BlockRef::next_possible_block_start(
block.addr() + min_outer_size, BlockRef::MIN_ALIGN);
size_t new_outer_size = next_block_start - block.addr();
if (block.outer_size() >= new_outer_size) {
optional<BlockRef> next = block.split(size);
// register the new block on successful split
if (next.has_value()) {
Block *next_block = *next;
Block *right = next_block->next();
if (right != nullptr && !right->used()) {
BlockRef next_block = *next;
BlockRef right = next_block.next();
// Since the original block was not the last block (the sentinel last
// block is never split), the split-off remainder block `next_block` is
// also not the last block. Thus, its next block `right` is guaranteed
// to be non-null.
LIBC_ASSERT(right && "right block must be non-null");
if (!right.used()) {
free_store.remove(right);
next_block->merge_next();
next_block.merge_next();
}
free_store.insert(next_block);
}
@ -209,10 +209,10 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
if (!is_valid_ptr(bytes))
return nullptr;
Block *block = Block::from_usable_space(bytes);
if (!block->used())
BlockRef block = BlockRef::from_usable_space(bytes);
if (!block.used())
return nullptr;
size_t old_size = block->inner_size();
size_t old_size = block.inner_size();
if (old_size >= size) {
shrink_in_place(block, size);

55
libc/src/__support/freestore.h
View file

@ -1,10 +1,15 @@
//===-- Interface for freestore ------------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Interface for freestore.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
@ -31,39 +36,39 @@ public:
/// Insert a free block. If the block is too small to be tracked, nothing
/// happens.
void insert(Block *block);
void insert(BlockRef block);
/// Remove a free block. If the block is too small to be tracked, nothing
/// happens.
void remove(Block *block);
void remove(BlockRef block);
/// Remove a best-fit free block that can contain the given size when
/// allocated. Returns nullptr if there is no such block.
Block *remove_best_fit(size_t size);
BlockRef remove_best_fit(size_t size);
private:
static constexpr size_t MIN_OUTER_SIZE =
align_up(sizeof(Block) + sizeof(FreeList::Node), Block::MIN_ALIGN);
static constexpr size_t MIN_LARGE_OUTER_SIZE =
align_up(sizeof(Block) + sizeof(FreeTrie::Node), Block::MIN_ALIGN);
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
static constexpr size_t MIN_LARGE_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeTrie::Node), BlockRef::MIN_ALIGN);
static constexpr size_t NUM_SMALL_SIZES =
(MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / Block::MIN_ALIGN;
(MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / BlockRef::MIN_ALIGN;
LIBC_INLINE static bool too_small(Block *block) {
return block->outer_size() < MIN_OUTER_SIZE;
LIBC_INLINE static bool too_small(BlockRef block) {
return block.outer_size() < MIN_OUTER_SIZE;
}
LIBC_INLINE static bool is_small(Block *block) {
return block->outer_size() < MIN_LARGE_OUTER_SIZE;
LIBC_INLINE static bool is_small(BlockRef block) {
return block.outer_size() < MIN_LARGE_OUTER_SIZE;
}
FreeList &small_list(Block *block);
FreeList &small_list(BlockRef block);
FreeList *find_best_small_fit(size_t size);
cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
FreeTrie large_trie;
};
LIBC_INLINE void FreeStore::insert(Block *block) {
LIBC_INLINE void FreeStore::insert(BlockRef block) {
if (too_small(block))
return;
if (is_small(block))
@ -72,35 +77,35 @@ LIBC_INLINE void FreeStore::insert(Block *block) {
large_trie.push(block);
}
LIBC_INLINE void FreeStore::remove(Block *block) {
LIBC_INLINE void FreeStore::remove(BlockRef block) {
if (too_small(block))
return;
if (is_small(block)) {
small_list(block).remove(
reinterpret_cast<FreeList::Node *>(block->usable_space()));
reinterpret_cast<FreeList::Node *>(block.usable_space()));
} else {
large_trie.remove(
reinterpret_cast<FreeTrie::Node *>(block->usable_space()));
large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
}
}
LIBC_INLINE Block *FreeStore::remove_best_fit(size_t size) {
LIBC_INLINE BlockRef FreeStore::remove_best_fit(size_t size) {
if (FreeList *list = find_best_small_fit(size)) {
Block *block = list->front();
BlockRef block = list->front();
list->pop();
return block;
}
if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
Block *block = best_fit->block();
BlockRef block = best_fit->block();
large_trie.remove(best_fit);
return block;
}
return nullptr;
return BlockRef();
}
LIBC_INLINE FreeList &FreeStore::small_list(Block *block) {
LIBC_INLINE FreeList &FreeStore::small_list(BlockRef block) {
LIBC_ASSERT(is_small(block) && "only legal for small blocks");
return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / Block::MIN_ALIGN];
return small_lists[(block.outer_size() - MIN_OUTER_SIZE) /
BlockRef::MIN_ALIGN];
}
LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {

17
libc/src/__support/freetrie.h
View file

@ -1,10 +1,15 @@
//===-- Interface for freetrie --------------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Interface for freetrie.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
#define LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
@ -96,7 +101,7 @@ public:
LIBC_INLINE bool empty() const { return !root; }
/// Push a block to the trie.
void push(Block *block);
void push(BlockRef block);
/// Remove a node from this trie node's free list.
void remove(Node *node);
@ -117,10 +122,10 @@ private:
SizeRange range;
};
LIBC_INLINE void FreeTrie::push(Block *block) {
LIBC_ASSERT(block->inner_size_free() >= sizeof(Node) &&
LIBC_INLINE void FreeTrie::push(BlockRef block) {
LIBC_ASSERT(block.inner_size_free() >= sizeof(Node) &&
"block too small to accomodate free trie node");
size_t size = block->inner_size();
size_t size = block.inner_size();
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
// Find the position in the tree to push to.
@ -139,7 +144,7 @@ LIBC_INLINE void FreeTrie::push(Block *block) {
}
}
Node *node = new (block->usable_space()) Node;
Node *node = new (block.usable_space()) Node;
FreeList list = *cur;
if (list.empty()) {
node->parent = parent;

430
libc/test/src/__support/block_test.cpp
View file

@ -1,10 +1,15 @@
//===-- Unittests for a block of memory -------------------------*- C++ -*-===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Unittests for a block of memory.
///
//===----------------------------------------------------------------------===//
#include <stddef.h>
#include "src/__support/CPP/array.h"
@ -14,7 +19,7 @@
#include "src/string/memcpy.h"
#include "test/UnitTest/Test.h"
using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::cpp::array;
using LIBC_NAMESPACE::cpp::bit_ceil;
using LIBC_NAMESPACE::cpp::byte;
@ -22,54 +27,53 @@ using LIBC_NAMESPACE::cpp::span;
TEST(LlvmLibcBlockTest, CanCreateSingleAlignedBlock) {
constexpr size_t kN = 1024;
alignas(Block::MIN_ALIGN) array<byte, kN> bytes;
alignas(BlockRef::MIN_ALIGN) array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
EXPECT_EQ(reinterpret_cast<uintptr_t>(block) % alignof(Block), size_t{0});
EXPECT_TRUE(block->is_usable_space_aligned(Block::MIN_ALIGN));
EXPECT_EQ(block.addr() % alignof(size_t), size_t{0});
EXPECT_TRUE(block.is_usable_space_aligned(BlockRef::MIN_ALIGN));
Block *last = block->next();
ASSERT_NE(last, static_cast<Block *>(nullptr));
EXPECT_EQ(reinterpret_cast<uintptr_t>(last) % alignof(Block), size_t{0});
BlockRef last = block.next();
ASSERT_NE(last.addr(), BlockRef().addr());
EXPECT_EQ(last.addr() % alignof(size_t), size_t{0});
EXPECT_EQ(last->outer_size(), sizeof(Block));
EXPECT_EQ(last->prev_free(), block);
EXPECT_TRUE(last->used());
EXPECT_EQ(last.outer_size(), BlockRef::HEADER_SIZE);
EXPECT_EQ(last.prev_free().addr(), block.addr());
EXPECT_TRUE(last.used());
size_t block_outer_size =
reinterpret_cast<uintptr_t>(last) - reinterpret_cast<uintptr_t>(block);
EXPECT_EQ(block->outer_size(), block_outer_size);
EXPECT_EQ(block->inner_size(),
block_outer_size - sizeof(Block) + Block::PREV_FIELD_SIZE);
EXPECT_EQ(block->prev_free(), static_cast<Block *>(nullptr));
EXPECT_FALSE(block->used());
size_t block_outer_size = last.addr() - block.addr();
EXPECT_EQ(block.outer_size(), block_outer_size);
EXPECT_EQ(block.inner_size(), block_outer_size - BlockRef::HEADER_SIZE +
BlockRef::PREV_FIELD_SIZE);
EXPECT_EQ(block.prev_free().addr(), BlockRef().addr());
EXPECT_FALSE(block.used());
}
TEST(LlvmLibcBlockTest, CanCreateUnalignedSingleBlock) {
constexpr size_t kN = 1024;
// Force alignment, so we can un-force it below
alignas(Block::MIN_ALIGN) array<byte, kN> bytes;
alignas(BlockRef::MIN_ALIGN) array<byte, kN> bytes;
span<byte> aligned(bytes);
auto result = Block::init(aligned.subspan(1));
auto result = BlockRef::init(aligned.subspan(1));
EXPECT_TRUE(result.has_value());
Block *block = *result;
EXPECT_EQ(reinterpret_cast<uintptr_t>(block) % alignof(Block), size_t{0});
EXPECT_TRUE(block->is_usable_space_aligned(Block::MIN_ALIGN));
BlockRef block = *result;
EXPECT_EQ(block.addr() % alignof(size_t), size_t{0});
EXPECT_TRUE(block.is_usable_space_aligned(BlockRef::MIN_ALIGN));
Block *last = block->next();
ASSERT_NE(last, static_cast<Block *>(nullptr));
EXPECT_EQ(reinterpret_cast<uintptr_t>(last) % alignof(Block), size_t{0});
BlockRef last = block.next();
ASSERT_NE(last.addr(), BlockRef().addr());
EXPECT_EQ(last.addr() % alignof(size_t), size_t{0});
}
TEST(LlvmLibcBlockTest, CannotCreateTooSmallBlock) {
array<byte, 2> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
EXPECT_FALSE(result.has_value());
}
@ -79,55 +83,55 @@ TEST(LlvmLibcBlockTest, CanSplitBlock) {
// Choose a split position such that the next block's usable space is 512
// bytes from this one's. This should be sufficient for any machine's
// alignment.
const size_t kSplitN = Block::inner_size(512);
const size_t kSplitN = BlockRef::inner_size(512);
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
auto *block1 = *result;
size_t orig_size = block1->outer_size();
BlockRef block1 = *result;
size_t orig_size = block1.outer_size();
result = block1->split(kSplitN);
result = block1.split(kSplitN);
ASSERT_TRUE(result.has_value());
auto *block2 = *result;
BlockRef block2 = *result;
EXPECT_EQ(block1->inner_size(), kSplitN);
EXPECT_EQ(block1->outer_size(),
kSplitN - Block::PREV_FIELD_SIZE + sizeof(Block));
EXPECT_EQ(block1.inner_size(), kSplitN);
EXPECT_EQ(block1.outer_size(),
kSplitN - BlockRef::PREV_FIELD_SIZE + BlockRef::HEADER_SIZE);
EXPECT_EQ(block2->outer_size(), orig_size - block1->outer_size());
EXPECT_FALSE(block2->used());
EXPECT_EQ(reinterpret_cast<uintptr_t>(block2) % alignof(Block), size_t{0});
EXPECT_TRUE(block2->is_usable_space_aligned(Block::MIN_ALIGN));
EXPECT_EQ(block2.outer_size(), orig_size - block1.outer_size());
EXPECT_FALSE(block2.used());
EXPECT_EQ(block2.addr() % alignof(size_t), size_t{0});
EXPECT_TRUE(block2.is_usable_space_aligned(BlockRef::MIN_ALIGN));
EXPECT_EQ(block1->next(), block2);
EXPECT_EQ(block2->prev_free(), block1);
EXPECT_EQ(block1.next().addr(), block2.addr());
EXPECT_EQ(block2.prev_free().addr(), block1.addr());
}
TEST(LlvmLibcBlockTest, CanSplitBlockUnaligned) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block1 = *result;
size_t orig_size = block1->outer_size();
BlockRef block1 = *result;
size_t orig_size = block1.outer_size();
constexpr size_t kSplitN = 513;
result = block1->split(kSplitN);
result = block1.split(kSplitN);
ASSERT_TRUE(result.has_value());
Block *block2 = *result;
BlockRef block2 = *result;
EXPECT_GE(block1->inner_size(), kSplitN);
EXPECT_GE(block1.inner_size(), kSplitN);
EXPECT_EQ(block2->outer_size(), orig_size - block1->outer_size());
EXPECT_FALSE(block2->used());
EXPECT_EQ(reinterpret_cast<uintptr_t>(block2) % alignof(Block), size_t{0});
EXPECT_TRUE(block2->is_usable_space_aligned(Block::MIN_ALIGN));
EXPECT_EQ(block2.outer_size(), orig_size - block1.outer_size());
EXPECT_FALSE(block2.used());
EXPECT_EQ(block2.addr() % alignof(size_t), size_t{0});
EXPECT_TRUE(block2.is_usable_space_aligned(BlockRef::MIN_ALIGN));
EXPECT_EQ(block1->next(), block2);
EXPECT_EQ(block2->prev_free(), block1);
EXPECT_EQ(block1.next().addr(), block2.addr());
EXPECT_EQ(block2.prev_free().addr(), block1.addr());
}
TEST(LlvmLibcBlockTest, CanSplitMidBlock) {
@ -135,9 +139,9 @@ TEST(LlvmLibcBlockTest, CanSplitMidBlock) {
// pointers get rewired properly.
// I.e.
// [[ BLOCK 1 ]]
// block1->split()
// block1.split()
// [[ BLOCK1 ]][[ BLOCK2 ]]
// block1->split()
// block1.split()
// [[ BLOCK1 ]][[ BLOCK3 ]][[ BLOCK2 ]]
constexpr size_t kN = 1024;
@ -145,33 +149,33 @@ TEST(LlvmLibcBlockTest, CanSplitMidBlock) {
constexpr size_t kSplit2 = 256;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block1 = *result;
BlockRef block1 = *result;
result = block1->split(kSplit1);
result = block1.split(kSplit1);
ASSERT_TRUE(result.has_value());
Block *block2 = *result;
BlockRef block2 = *result;
result = block1->split(kSplit2);
result = block1.split(kSplit2);
ASSERT_TRUE(result.has_value());
Block *block3 = *result;
BlockRef block3 = *result;
EXPECT_EQ(block1->next(), block3);
EXPECT_EQ(block3->prev_free(), block1);
EXPECT_EQ(block3->next(), block2);
EXPECT_EQ(block2->prev_free(), block3);
EXPECT_EQ(block1.next().addr(), block3.addr());
EXPECT_EQ(block3.prev_free().addr(), block1.addr());
EXPECT_EQ(block3.next().addr(), block2.addr());
EXPECT_EQ(block2.prev_free().addr(), block3.addr());
}
TEST(LlvmLibcBlockTest, CannotSplitTooSmallBlock) {
constexpr size_t kN = 64;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
result = block->split(block->inner_size() + 1);
result = block.split(block.inner_size() + 1);
ASSERT_FALSE(result.has_value());
}
@ -179,11 +183,11 @@ TEST(LlvmLibcBlockTest, CannotSplitBlockWithoutHeaderSpace) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
result = block->split(block->inner_size() - sizeof(Block) + 1);
result = block.split(block.inner_size() - BlockRef::HEADER_SIZE + 1);
ASSERT_FALSE(result.has_value());
}
@ -192,11 +196,11 @@ TEST(LlvmLibcBlockTest, CannotMakeBlockLargerInSplit) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
result = block->split(block->inner_size() + 1);
result = block.split(block.inner_size() + 1);
ASSERT_FALSE(result.has_value());
}
@ -205,13 +209,13 @@ TEST(LlvmLibcBlockTest, CanMakeMinimalSizeFirstBlock) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
result = block->split(0);
result = block.split(0);
ASSERT_TRUE(result.has_value());
EXPECT_LE(block->outer_size(), sizeof(Block) + Block::MIN_ALIGN);
EXPECT_LE(block.outer_size(), BlockRef::HEADER_SIZE + BlockRef::MIN_ALIGN);
}
TEST(LlvmLibcBlockTest, CanMakeMinimalSizeSecondBlock) {
@ -219,33 +223,34 @@ TEST(LlvmLibcBlockTest, CanMakeMinimalSizeSecondBlock) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block1 = *result;
BlockRef block1 = *result;
result = block1->split(Block::prev_possible_block_start(
reinterpret_cast<uintptr_t>(block1->next())) -
reinterpret_cast<uintptr_t>(block1->usable_space()) +
Block::PREV_FIELD_SIZE);
result =
block1.split(BlockRef::prev_possible_block_start(block1.next().addr()) -
reinterpret_cast<uintptr_t>(block1.usable_space()) +
BlockRef::PREV_FIELD_SIZE);
ASSERT_TRUE(result.has_value());
EXPECT_LE((*result)->outer_size(), sizeof(Block) + Block::MIN_ALIGN);
EXPECT_LE((*result).outer_size(),
BlockRef::HEADER_SIZE + BlockRef::MIN_ALIGN);
}
TEST(LlvmLibcBlockTest, CanMarkBlockUsed) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
size_t orig_size = block->outer_size();
BlockRef block = *result;
size_t orig_size = block.outer_size();
block->mark_used();
EXPECT_TRUE(block->used());
EXPECT_EQ(block->outer_size(), orig_size);
block.mark_used();
EXPECT_TRUE(block.used());
EXPECT_EQ(block.outer_size(), orig_size);
block->mark_free();
EXPECT_FALSE(block->used());
block.mark_free();
EXPECT_FALSE(block.used());
}
TEST(LlvmLibcBlockTest, CanSplitUsedBlock) {
@ -253,19 +258,19 @@ TEST(LlvmLibcBlockTest, CanSplitUsedBlock) {
constexpr size_t kSplitN = 512;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
block->mark_used();
result = block->split(kSplitN);
block.mark_used();
result = block.split(kSplitN);
ASSERT_TRUE(result.has_value());
Block *new_block = *result;
BlockRef new_block = *result;
EXPECT_TRUE(block->used());
EXPECT_FALSE(new_block->used());
EXPECT_EQ(block->next(), new_block);
EXPECT_EQ(new_block->prev_free(), static_cast<Block *>(nullptr));
EXPECT_TRUE(block.used());
EXPECT_FALSE(new_block.used());
EXPECT_EQ(block.next().addr(), new_block.addr());
EXPECT_EQ(new_block.prev_free().addr(), BlockRef().addr());
}
TEST(LlvmLibcBlockTest, CanMergeWithNextBlock) {
@ -275,25 +280,25 @@ TEST(LlvmLibcBlockTest, CanMergeWithNextBlock) {
constexpr size_t kSplit1 = 512;
constexpr size_t kSplit2 = 256;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block1 = *result;
size_t total_size = block1->outer_size();
BlockRef block1 = *result;
size_t total_size = block1.outer_size();
result = block1->split(kSplit1);
result = block1.split(kSplit1);
ASSERT_TRUE(result.has_value());
result = block1->split(kSplit2);
size_t block1_size = block1->outer_size();
result = block1.split(kSplit2);
size_t block1_size = block1.outer_size();
ASSERT_TRUE(result.has_value());
Block *block3 = *result;
BlockRef block3 = *result;
EXPECT_TRUE(block3->merge_next());
EXPECT_TRUE(block3.merge_next());
EXPECT_EQ(block1->next(), block3);
EXPECT_EQ(block3->prev_free(), block1);
EXPECT_EQ(block1->outer_size(), block1_size);
EXPECT_EQ(block3->outer_size(), total_size - block1->outer_size());
EXPECT_EQ(block1.next().addr(), block3.addr());
EXPECT_EQ(block3.prev_free().addr(), block1.addr());
EXPECT_EQ(block1.outer_size(), block1_size);
EXPECT_EQ(block3.outer_size(), total_size - block1.outer_size());
}
TEST(LlvmLibcBlockTest, CannotMergeWithFirstOrLastBlock) {
@ -301,16 +306,16 @@ TEST(LlvmLibcBlockTest, CannotMergeWithFirstOrLastBlock) {
constexpr size_t kSplitN = 512;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block1 = *result;
BlockRef block1 = *result;
// Do a split, just to check that the checks on next/prev are different...
result = block1->split(kSplitN);
result = block1.split(kSplitN);
ASSERT_TRUE(result.has_value());
Block *block2 = *result;
BlockRef block2 = *result;
EXPECT_FALSE(block2->merge_next());
EXPECT_FALSE(block2.merge_next());
}
TEST(LlvmLibcBlockTest, CannotMergeUsedBlock) {
@ -318,40 +323,40 @@ TEST(LlvmLibcBlockTest, CannotMergeUsedBlock) {
constexpr size_t kSplitN = 512;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
// Do a split, just to check that the checks on next/prev are different...
result = block->split(kSplitN);
result = block.split(kSplitN);
ASSERT_TRUE(result.has_value());
block->mark_used();
EXPECT_FALSE(block->merge_next());
block.mark_used();
EXPECT_FALSE(block.merge_next());
}
TEST(LlvmLibcBlockTest, CanGetBlockFromUsableSpace) {
array<byte, 1024> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block1 = *result;
BlockRef block1 = *result;
void *ptr = block1->usable_space();
Block *block2 = Block::from_usable_space(ptr);
EXPECT_EQ(block1, block2);
void *ptr = block1.usable_space();
BlockRef block2 = BlockRef::from_usable_space(ptr);
EXPECT_EQ(block1.addr(), block2.addr());
}
TEST(LlvmLibcBlockTest, CanGetConstBlockFromUsableSpace) {
constexpr size_t kN = 1024;
array<byte, kN> bytes{};
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
const Block *block1 = *result;
BlockRef block1 = *result;
const void *ptr = block1->usable_space();
const Block *block2 = Block::from_usable_space(ptr);
EXPECT_EQ(block1, block2);
const void *ptr = block1.usable_space();
BlockRef block2 = BlockRef::from_usable_space(ptr);
EXPECT_EQ(block1.addr(), block2.addr());
}
TEST(LlvmLibcBlockTest, Allocate) {
@ -360,30 +365,30 @@ TEST(LlvmLibcBlockTest, Allocate) {
// Ensure we can allocate everything up to the block size within this block.
for (size_t i = 0; i < kN; ++i) {
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
if (i > block->inner_size())
if (i > block.inner_size())
continue;
auto info = Block::allocate(block, Block::MIN_ALIGN, i);
EXPECT_NE(info.block, static_cast<Block *>(nullptr));
auto info = BlockRef::allocate(block, BlockRef::MIN_ALIGN, i);
EXPECT_NE(info.block.addr(), BlockRef().addr());
}
// Ensure we can allocate a byte at every guaranteeable alignment.
for (size_t i = 1; i < kN / Block::MIN_ALIGN; ++i) {
for (size_t i = 1; i < kN / BlockRef::MIN_ALIGN; ++i) {
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
size_t alignment = i * Block::MIN_ALIGN;
if (Block::min_size_for_allocation(alignment, 1) > block->inner_size())
size_t alignment = i * BlockRef::MIN_ALIGN;
if (BlockRef::min_size_for_allocation(alignment, 1) > block.inner_size())
continue;
auto info = Block::allocate(block, alignment, 1);
EXPECT_NE(info.block, static_cast<Block *>(nullptr));
auto info = BlockRef::allocate(block, alignment, 1);
EXPECT_NE(info.block.addr(), BlockRef().addr());
}
}
@ -391,98 +396,97 @@ TEST(LlvmLibcBlockTest, AllocateAlreadyAligned) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
uintptr_t orig_end = reinterpret_cast<uintptr_t>(block) + block->outer_size();
BlockRef block = *result;
uintptr_t orig_end = block.addr() + block.outer_size();
constexpr size_t SIZE = Block::PREV_FIELD_SIZE + 1;
constexpr size_t SIZE = BlockRef::PREV_FIELD_SIZE + 1;
auto [aligned_block, prev, next] =
Block::allocate(block, Block::MIN_ALIGN, SIZE);
BlockRef::allocate(block, BlockRef::MIN_ALIGN, SIZE);
// Since this is already aligned, there should be no previous block.
EXPECT_EQ(prev, static_cast<Block *>(nullptr));
EXPECT_EQ(prev.addr(), BlockRef().addr());
// Ensure we the block is aligned and large enough.
EXPECT_NE(aligned_block, static_cast<Block *>(nullptr));
EXPECT_TRUE(aligned_block->is_usable_space_aligned(Block::MIN_ALIGN));
EXPECT_GE(aligned_block->inner_size(), SIZE);
EXPECT_NE(aligned_block.addr(), BlockRef().addr());
EXPECT_TRUE(aligned_block.is_usable_space_aligned(BlockRef::MIN_ALIGN));
EXPECT_GE(aligned_block.inner_size(), SIZE);
// Check the next block.
EXPECT_NE(next, static_cast<Block *>(nullptr));
EXPECT_EQ(aligned_block->next(), next);
EXPECT_EQ(reinterpret_cast<uintptr_t>(next) + next->outer_size(), orig_end);
EXPECT_NE(next.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.next().addr(), next.addr());
EXPECT_EQ(next.addr() + next.outer_size(), orig_end);
}
TEST(LlvmLibcBlockTest, AllocateNeedsAlignment) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
uintptr_t orig_end = reinterpret_cast<uintptr_t>(block) + block->outer_size();
uintptr_t orig_end = block.addr() + block.outer_size();
// Now pick an alignment such that the usable space is not already aligned to
// it. We want to explicitly test that the block will split into one before
// it.
size_t alignment = Block::MIN_ALIGN;
while (block->is_usable_space_aligned(alignment))
alignment += Block::MIN_ALIGN;
size_t alignment = BlockRef::MIN_ALIGN;
while (block.is_usable_space_aligned(alignment))
alignment += BlockRef::MIN_ALIGN;
auto [aligned_block, prev, next] = Block::allocate(block, alignment, 10);
auto [aligned_block, prev, next] = BlockRef::allocate(block, alignment, 10);
// Check the previous block was created appropriately. Since this block is the
// first block, a new one should be made before this.
EXPECT_NE(prev, static_cast<Block *>(nullptr));
EXPECT_EQ(aligned_block->prev_free(), prev);
EXPECT_EQ(prev->next(), aligned_block);
EXPECT_EQ(prev->outer_size(), reinterpret_cast<uintptr_t>(aligned_block) -
reinterpret_cast<uintptr_t>(prev));
EXPECT_NE(prev.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.prev_free().addr(), prev.addr());
EXPECT_EQ(prev.next().addr(), aligned_block.addr());
EXPECT_EQ(prev.outer_size(), aligned_block.addr() - prev.addr());
// Ensure we the block is aligned and the size we expect.
EXPECT_NE(next, static_cast<Block *>(nullptr));
EXPECT_TRUE(aligned_block->is_usable_space_aligned(alignment));
EXPECT_NE(next.addr(), BlockRef().addr());
EXPECT_TRUE(aligned_block.is_usable_space_aligned(alignment));
// Check the next block.
EXPECT_NE(next, static_cast<Block *>(nullptr));
EXPECT_EQ(aligned_block->next(), next);
EXPECT_EQ(reinterpret_cast<uintptr_t>(next) + next->outer_size(), orig_end);
EXPECT_NE(next.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.next().addr(), next.addr());
EXPECT_EQ(next.addr() + next.outer_size(), orig_end);
}
TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
// Split the block roughly halfway and work on the second half.
auto result2 = block->split(kN / 2);
auto result2 = block.split(kN / 2);
ASSERT_TRUE(result2.has_value());
Block *newblock = *result2;
ASSERT_EQ(newblock->prev_free(), block);
size_t old_prev_size = block->outer_size();
BlockRef newblock = *result2;
ASSERT_EQ(newblock.prev_free().addr(), block.addr());
size_t old_prev_size = block.outer_size();
// Now pick an alignment such that the usable space is not already aligned to
// it. We want to explicitly test that the block will split into one before
// it.
size_t alignment = Block::MIN_ALIGN;
while (newblock->is_usable_space_aligned(alignment))
alignment += Block::MIN_ALIGN;
size_t alignment = BlockRef::MIN_ALIGN;
while (newblock.is_usable_space_aligned(alignment))
alignment += BlockRef::MIN_ALIGN;
// Ensure we can allocate in the new block.
auto [aligned_block, prev, next] = Block::allocate(newblock, alignment, 1);
auto [aligned_block, prev, next] = BlockRef::allocate(newblock, alignment, 1);
// Now there should be no new previous block. Instead, the padding we did
// create should be merged into the original previous block.
EXPECT_EQ(prev, static_cast<Block *>(nullptr));
EXPECT_EQ(aligned_block->prev_free(), block);
EXPECT_EQ(block->next(), aligned_block);
EXPECT_GT(block->outer_size(), old_prev_size);
EXPECT_EQ(prev.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.prev_free().addr(), block.addr());
EXPECT_EQ(block.next().addr(), aligned_block.addr());
EXPECT_GT(block.outer_size(), old_prev_size);
}
TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) {
@ -493,40 +497,40 @@ TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) {
constexpr size_t kN = 1024;
array<byte, kN> bytes;
auto result = Block::init(bytes);
auto result = BlockRef::init(bytes);
ASSERT_TRUE(result.has_value());
Block *block = *result;
BlockRef block = *result;
Block *orig_block = block;
size_t orig_size = orig_block->outer_size();
BlockRef orig_block = block;
size_t orig_size = orig_block.outer_size();
Block *last = block->next();
BlockRef last = block.next();
ASSERT_EQ(block->prev_free(), static_cast<Block *>(nullptr));
ASSERT_EQ(block.prev_free().addr(), BlockRef().addr());
// Now pick an alignment such that the usable space is not already aligned to
// it. We want to explicitly test that the block will split into one before
// it.
size_t alignment = Block::MIN_ALIGN;
while (block->is_usable_space_aligned(alignment))
alignment += Block::MIN_ALIGN;
size_t alignment = BlockRef::MIN_ALIGN;
while (block.is_usable_space_aligned(alignment))
alignment += BlockRef::MIN_ALIGN;
auto [aligned_block, prev, next] = Block::allocate(block, alignment, 1);
auto [aligned_block, prev, next] = BlockRef::allocate(block, alignment, 1);
// Check we have the appropriate blocks.
ASSERT_NE(prev, static_cast<Block *>(nullptr));
ASSERT_EQ(aligned_block->prev_free(), prev);
EXPECT_NE(next, static_cast<Block *>(nullptr));
EXPECT_EQ(aligned_block->next(), next);
EXPECT_EQ(next->next(), last);
ASSERT_NE(prev.addr(), BlockRef().addr());
ASSERT_EQ(aligned_block.prev_free().addr(), prev.addr());
EXPECT_NE(next.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.next().addr(), next.addr());
EXPECT_EQ(next.next().addr(), last.addr());
// Now check for successful merges.
EXPECT_TRUE(prev->merge_next());
EXPECT_EQ(prev->next(), next);
EXPECT_TRUE(prev->merge_next());
EXPECT_EQ(prev->next(), last);
EXPECT_TRUE(prev.merge_next());
EXPECT_EQ(prev.next().addr(), next.addr());
EXPECT_TRUE(prev.merge_next());
EXPECT_EQ(prev.next().addr(), last.addr());
// We should have the original buffer.
EXPECT_EQ(prev, orig_block);
EXPECT_EQ(prev->outer_size(), orig_size);
EXPECT_EQ(prev.addr(), orig_block.addr());
EXPECT_EQ(prev.outer_size(), orig_size);
}

13
libc/test/src/__support/freelist_heap_test.cpp
View file

@ -1,10 +1,15 @@
//===-- Unittests for freelist_heap ---------------------------------------===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Unittests for freelist_heap.
///
//===----------------------------------------------------------------------===//
#include "src/__support/CPP/span.h"
#include "src/__support/freelist_heap.h"
@ -23,7 +28,7 @@ _end:
__llvm_libc_heap_limit:
)");
using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
@ -124,12 +129,12 @@ TEST_FOR_EACH_ALLOCATOR(ReturnedPointersAreAligned, 2048) {
void *ptr1 = allocator.allocate(1);
uintptr_t ptr1_start = reinterpret_cast<uintptr_t>(ptr1);
EXPECT_EQ(ptr1_start % Block::MIN_ALIGN, static_cast<size_t>(0));
EXPECT_EQ(ptr1_start % BlockRef::MIN_ALIGN, static_cast<size_t>(0));
void *ptr2 = allocator.allocate(1);
uintptr_t ptr2_start = reinterpret_cast<uintptr_t>(ptr2);
EXPECT_EQ(ptr2_start % Block::MIN_ALIGN, static_cast<size_t>(0));
EXPECT_EQ(ptr2_start % BlockRef::MIN_ALIGN, static_cast<size_t>(0));
}
TEST_FOR_EACH_ALLOCATOR(CanRealloc, 2048) {

29
libc/test/src/__support/freelist_test.cpp
View file

@ -1,53 +1,58 @@
//===-- Unittests for a freelist --------------------------------*- C++ -*-===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Unittests for a freelist.
///
//===----------------------------------------------------------------------===//
#include <stddef.h>
#include "src/__support/freelist.h"
#include "test/UnitTest/Test.h"
using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeList;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
TEST(LlvmLibcFreeList, FreeList) {
byte mem[1024];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *block1 = *maybeBlock;
BlockRef block1 = *maybeBlock;
maybeBlock = block1->split(128);
maybeBlock = block1.split(128);
ASSERT_TRUE(maybeBlock.has_value());
Block *block2 = *maybeBlock;
BlockRef block2 = *maybeBlock;
maybeBlock = block2->split(128);
maybeBlock = block2.split(128);
ASSERT_TRUE(maybeBlock.has_value());
FreeList list;
list.push(block1);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front(), block1);
EXPECT_EQ(list.front().addr(), block1.addr());
list.push(block2);
EXPECT_EQ(list.front(), block1);
EXPECT_EQ(list.front().addr(), block1.addr());
list.pop();
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front(), block2);
EXPECT_EQ(list.front().addr(), block2.addr());
list.pop();
ASSERT_TRUE(list.empty());
list.push(block1);
list.push(block2);
list.remove(reinterpret_cast<FreeList::Node *>(block2->usable_space()));
EXPECT_EQ(list.front(), block1);
list.remove(reinterpret_cast<FreeList::Node *>(block2.usable_space()));
EXPECT_EQ(list.front().addr(), block1.addr());
list.pop();
ASSERT_TRUE(list.empty());
}

72
libc/test/src/__support/freestore_test.cpp
View file

@ -1,17 +1,22 @@
//===-- Unittests for a freestore -------------------------------*- C++ -*-===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Unittests for a freestore.
///
//===----------------------------------------------------------------------===//
#include <stddef.h>
#include "src/__support/freestore.h"
#include "test/UnitTest/Test.h"
using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeList;
using LIBC_NAMESPACE::FreeStore;
using LIBC_NAMESPACE::FreeTrie;
@ -21,44 +26,46 @@ using LIBC_NAMESPACE::cpp::optional;
// Inserting or removing blocks too small to be tracked does nothing.
TEST(LlvmLibcFreeStore, TooSmall) {
byte mem[1024];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *too_small = *maybeBlock;
maybeBlock = too_small->split(Block::PREV_FIELD_SIZE);
BlockRef too_small = *maybeBlock;
maybeBlock = too_small.split(BlockRef::PREV_FIELD_SIZE);
ASSERT_TRUE(maybeBlock.has_value());
// On platforms with high alignment the smallest legal block may be large
// enough for a node.
if (too_small->outer_size() >= sizeof(Block) + sizeof(FreeList::Node))
if (too_small.outer_size() >= BlockRef::HEADER_SIZE + sizeof(FreeList::Node))
return;
Block *remainder = *maybeBlock;
BlockRef remainder = *maybeBlock;
FreeStore store;
store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
EXPECT_EQ(store.remove_best_fit(too_small->inner_size()), remainder);
EXPECT_EQ(store.remove_best_fit(too_small.inner_size()).addr(),
remainder.addr());
store.remove(too_small);
}
TEST(LlvmLibcFreeStore, RemoveBestFit) {
byte mem[1024];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *smallest = *maybeBlock;
maybeBlock = smallest->split(sizeof(FreeList::Node) + Block::PREV_FIELD_SIZE);
BlockRef smallest = *maybeBlock;
maybeBlock =
smallest.split(sizeof(FreeList::Node) + BlockRef::PREV_FIELD_SIZE);
ASSERT_TRUE(maybeBlock.has_value());
Block *largest_small = *maybeBlock;
maybeBlock = largest_small->split(sizeof(FreeTrie::Node) +
Block::PREV_FIELD_SIZE - Block::MIN_ALIGN);
BlockRef largest_small = *maybeBlock;
maybeBlock = largest_small.split(
sizeof(FreeTrie::Node) + BlockRef::PREV_FIELD_SIZE - BlockRef::MIN_ALIGN);
ASSERT_TRUE(maybeBlock.has_value());
if (largest_small->inner_size() == smallest->inner_size())
if (largest_small.inner_size() == smallest.inner_size())
largest_small = smallest;
ASSERT_GE(largest_small->inner_size(), smallest->inner_size());
ASSERT_GE(largest_small.inner_size(), smallest.inner_size());
Block *remainder = *maybeBlock;
BlockRef remainder = *maybeBlock;
FreeStore store;
store.set_range({0, 4096});
@ -68,32 +75,37 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
store.insert(remainder);
// Find exact match for smallest.
ASSERT_EQ(store.remove_best_fit(smallest->inner_size()), smallest);
ASSERT_EQ(store.remove_best_fit(smallest.inner_size()).addr(),
smallest.addr());
store.insert(smallest);
// Find exact match for largest.
ASSERT_EQ(store.remove_best_fit(largest_small->inner_size()), largest_small);
ASSERT_EQ(store.remove_best_fit(largest_small.inner_size()).addr(),
largest_small.addr());
store.insert(largest_small);
// Search small list for best fit.
Block *next_smallest = largest_small == smallest ? remainder : largest_small;
ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), next_smallest);
BlockRef next_smallest =
largest_small == smallest ? remainder : largest_small;
ASSERT_EQ(store.remove_best_fit(smallest.inner_size() + 1).addr(),
next_smallest.addr());
store.insert(next_smallest);
// Continue search for best fit to large blocks.
EXPECT_EQ(store.remove_best_fit(largest_small->inner_size() + 1), remainder);
EXPECT_EQ(store.remove_best_fit(largest_small.inner_size() + 1).addr(),
remainder.addr());
}
TEST(LlvmLibcFreeStore, Remove) {
byte mem[1024];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *small = *maybeBlock;
maybeBlock = small->split(sizeof(FreeList::Node) + Block::PREV_FIELD_SIZE);
BlockRef small = *maybeBlock;
maybeBlock = small.split(sizeof(FreeList::Node) + BlockRef::PREV_FIELD_SIZE);
ASSERT_TRUE(maybeBlock.has_value());
Block *remainder = *maybeBlock;
BlockRef remainder = *maybeBlock;
FreeStore store;
store.set_range({0, 4096});
@ -101,9 +113,9 @@ TEST(LlvmLibcFreeStore, Remove) {
store.insert(remainder);
store.remove(remainder);
ASSERT_EQ(store.remove_best_fit(remainder->inner_size()),
static_cast<Block *>(nullptr));
ASSERT_EQ(store.remove_best_fit(remainder.inner_size()).addr(),
BlockRef().addr());
store.remove(small);
ASSERT_EQ(store.remove_best_fit(small->inner_size()),
static_cast<Block *>(nullptr));
ASSERT_EQ(store.remove_best_fit(small.inner_size()).addr(),
BlockRef().addr());
}

87
libc/test/src/__support/freetrie_test.cpp
View file

@ -1,17 +1,22 @@
//===-- Unittests for a freetrie --------------------------------*- C++ -*-===//
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Unittests for a freetrie.
///
//===----------------------------------------------------------------------===//
#include <stddef.h>
#include "src/__support/freetrie.h"
#include "test/UnitTest/Test.h"
using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeTrie;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
@ -21,65 +26,67 @@ TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
EXPECT_EQ(trie.find_best_fit(123), static_cast<FreeTrie::Node *>(nullptr));
byte mem[1024];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *block = *maybeBlock;
BlockRef block = *maybeBlock;
trie.push(block);
FreeTrie::Node *root = trie.find_best_fit(0);
ASSERT_EQ(root->block(), block);
EXPECT_EQ(trie.find_best_fit(block->inner_size() - 1), root);
EXPECT_EQ(trie.find_best_fit(block->inner_size()), root);
EXPECT_EQ(trie.find_best_fit(block->inner_size() + 1),
ASSERT_EQ(root->block().addr(), block.addr());
EXPECT_EQ(trie.find_best_fit(block.inner_size() - 1), root);
EXPECT_EQ(trie.find_best_fit(block.inner_size()), root);
EXPECT_EQ(trie.find_best_fit(block.inner_size() + 1),
static_cast<FreeTrie::Node *>(nullptr));
EXPECT_EQ(trie.find_best_fit(4095), static_cast<FreeTrie::Node *>(nullptr));
}
TEST(LlvmLibcFreeTrie, FindBestFitLower) {
byte mem[4096];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *lower = *maybeBlock;
maybeBlock = lower->split(512);
BlockRef lower = *maybeBlock;
maybeBlock = lower.split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block *root = *maybeBlock;
BlockRef root = *maybeBlock;
FreeTrie trie({0, 4096});
trie.push(root);
trie.push(lower);
EXPECT_EQ(trie.find_best_fit(0)->block(), lower);
EXPECT_EQ(trie.find_best_fit(0)->block().addr(), lower.addr());
}
TEST(LlvmLibcFreeTrie, FindBestFitUpper) {
byte mem[4096];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *root = *maybeBlock;
maybeBlock = root->split(512);
BlockRef root = *maybeBlock;
maybeBlock = root.split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block *upper = *maybeBlock;
BlockRef upper = *maybeBlock;
FreeTrie trie({0, 4096});
trie.push(root);
trie.push(upper);
EXPECT_EQ(trie.find_best_fit(root->inner_size() + 1)->block(), upper);
EXPECT_EQ(trie.find_best_fit(root.inner_size() + 1)->block().addr(),
upper.addr());
// The upper subtrie should be skipped if it could not contain a better fit.
EXPECT_EQ(trie.find_best_fit(root->inner_size() - 1)->block(), root);
EXPECT_EQ(trie.find_best_fit(root.inner_size() - 1)->block().addr(),
root.addr());
}
TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
byte mem[4096];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *root = *maybeBlock;
maybeBlock = root->split(1024);
BlockRef root = *maybeBlock;
maybeBlock = root.split(1024);
ASSERT_TRUE(maybeBlock.has_value());
Block *lower = *maybeBlock;
maybeBlock = lower->split(128);
BlockRef lower = *maybeBlock;
maybeBlock = lower.split(128);
ASSERT_TRUE(maybeBlock.has_value());
Block *upper = *maybeBlock;
BlockRef upper = *maybeBlock;
FreeTrie trie({0, 4096});
trie.push(root);
@ -87,30 +94,30 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
trie.push(upper);
// The lower subtrie is examined first.
EXPECT_EQ(trie.find_best_fit(0)->block(), lower);
EXPECT_EQ(trie.find_best_fit(0)->block().addr(), lower.addr());
// The upper subtrie is examined if there are no fits found in the upper
// subtrie.
EXPECT_EQ(trie.find_best_fit(2048)->block(), upper);
EXPECT_EQ(trie.find_best_fit(2048)->block().addr(), upper.addr());
}
TEST(LlvmLibcFreeTrie, Remove) {
byte mem[4096];
optional<Block *> maybeBlock = Block::init(mem);
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block *small1 = *maybeBlock;
maybeBlock = small1->split(512);
BlockRef small1 = *maybeBlock;
maybeBlock = small1.split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block *small2 = *maybeBlock;
maybeBlock = small2->split(512);
BlockRef small2 = *maybeBlock;
maybeBlock = small2.split(512);
ASSERT_TRUE(maybeBlock.has_value());
ASSERT_EQ(small1->inner_size(), small2->inner_size());
Block *large = *maybeBlock;
ASSERT_EQ(small1.inner_size(), small2.inner_size());
BlockRef large = *maybeBlock;
// Removing the root empties the trie.
FreeTrie trie({0, 4096});
trie.push(large);
FreeTrie::Node *large_node = trie.find_best_fit(0);
ASSERT_EQ(large_node->block(), large);
ASSERT_EQ(large_node->block().addr(), large.addr());
trie.remove(large_node);
ASSERT_TRUE(trie.empty());
@ -118,8 +125,10 @@ TEST(LlvmLibcFreeTrie, Remove) {
trie.push(small1);
trie.push(small2);
trie.push(large);
trie.remove(trie.find_best_fit(small1->inner_size()));
EXPECT_EQ(trie.find_best_fit(large->inner_size())->block(), large);
trie.remove(trie.find_best_fit(small1->inner_size()));
EXPECT_EQ(trie.find_best_fit(large->inner_size())->block(), large);
trie.remove(trie.find_best_fit(small1.inner_size()));
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
trie.remove(trie.find_best_fit(small1.inner_size()));
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
}