1 #ifndef HALIDE_RUNTIME_BLOCK_ALLOCATOR_H 2 #define HALIDE_RUNTIME_BLOCK_ALLOCATOR_H 4 #include "../HalideRuntime.h" 5 #include "../printer.h" 62 bool collect(
void *user_context);
63 int release(
void *user_context);
64 int destroy(
void *user_context);
87 int destroy_region_allocator(
void *user_context,
RegionAllocator *region_allocator);
99 int release_block_entry(
void *user_context,
BlockEntry *block_entry);
102 int destroy_block_entry(
void *user_context,
BlockEntry *block_entry);
105 int alloc_memory_block(
void *user_context,
BlockResource *block);
108 int free_memory_block(
void *user_context,
BlockResource *block);
111 size_t constrain_requested_size(
size_t size)
const;
129 if (result ==
nullptr) {
130 error(user_context) <<
"BlockAllocator: Failed to create instance! Out of memory\n";
134 result->initialize(user_context, cfg, allocators);
141 instance->
destroy(user_context);
146 void BlockAllocator::initialize(
void *user_context,
const Config &cfg,
const MemoryAllocators &ma) {
156 #ifdef DEBUG_RUNTIME_INTERNAL 157 debug(user_context) <<
"BlockAllocator: Reserve (" 158 <<
"user_context=" << (
void *)(user_context) <<
" " 161 <<
"dedicated=" << (request.
dedicated ?
"true" :
"false") <<
" " 167 BlockEntry *block_entry = reserve_block_entry(user_context, request);
168 if (block_entry ==
nullptr) {
169 error(user_context) <<
"BlockAllocator: Failed to allocate new empty block of requested size (" 180 if (result ==
nullptr) {
183 block_entry = create_block_entry(user_context, request);
184 if (block_entry ==
nullptr) {
185 error(user_context) <<
"BlockAllocator: Out of memory! Failed to allocate empty block of size (" 192 block->
allocator = create_region_allocator(user_context, block);
195 result = reserve_memory_region(user_context, block->
allocator, request);
201 if (memory_region ==
nullptr) {
205 if (allocator ==
nullptr) {
208 return allocator->
release(user_context, memory_region);
212 if (memory_region ==
nullptr) {
216 if (allocator ==
nullptr) {
219 return allocator->
reclaim(user_context, memory_region);
223 if (memory_region ==
nullptr) {
227 if (allocator ==
nullptr) {
230 return allocator->
retain(user_context, memory_region);
236 while (block_entry !=
nullptr) {
240 block_entry = prev_entry;
244 #ifdef DEBUG_RUNTIME_INTERNAL 250 #ifdef DEBUG_RUNTIME_INTERNAL 251 debug(user_context) <<
"Collected block (" 252 <<
"block=" << (
void *)block <<
" " 259 destroy_block_entry(user_context, block_entry);
263 block_entry = prev_entry;
270 while (block_entry !=
nullptr) {
272 release_block_entry(user_context, block_entry);
273 block_entry = prev_entry;
280 while (block_entry !=
nullptr) {
282 destroy_block_entry(user_context, block_entry);
283 block_entry = prev_entry;
285 block_list.
destroy(user_context);
291 if (result ==
nullptr) {
292 #ifdef DEBUG_RUNTIME_INTERNAL 293 debug(user_context) <<
"BlockAllocator: Failed to allocate region of size (" 298 if (allocator->
collect(user_context)) {
299 result = allocator->
reserve(user_context, request);
305 bool BlockAllocator::is_block_suitable_for_request(
void *user_context,
const BlockResource *block,
const MemoryRequest &request)
const {
306 if (!is_compatible_block(block, request.properties)) {
307 #ifdef DEBUG_RUNTIME_INTERNAL 308 debug(user_context) <<
"BlockAllocator: skipping block ... incompatible properties! (" 309 <<
"block_resource=" << (
void *)block <<
" " 311 <<
"block_reserved=" << (
uint32_t)block->reserved <<
" " 315 <<
"request_size=" << (
uint32_t)request.size <<
" " 324 if (request.dedicated && (block->reserved > 0)) {
325 #ifdef DEBUG_RUNTIME_INTERNAL 326 debug(user_context) <<
"BlockAllocator: skipping block ... can be used for dedicated allocation! (" 327 <<
"block_resource=" << (
void *)block <<
" " 328 <<
"block_size=" << (
uint32_t)block->memory.size <<
" " 329 <<
"block_reserved=" << (
uint32_t)block->reserved <<
")";
334 }
else if (block->memory.dedicated && (block->reserved > 0)) {
335 #ifdef DEBUG_RUNTIME_INTERNAL 336 debug(user_context) <<
"BlockAllocator: skipping block ... already dedicated to an allocation! (" 337 <<
"block_resource=" << (
void *)block <<
" " 338 <<
"block_size=" << (
uint32_t)block->memory.size <<
" " 339 <<
"block_reserved=" << (
uint32_t)block->reserved <<
")";
345 size_t available = (block->memory.size - block->reserved);
346 if (available >= request.size) {
353 BlockAllocator::BlockEntry *
354 BlockAllocator::find_block_entry(
void *user_context,
const MemoryRequest &request) {
355 BlockEntry *block_entry = block_list.
back();
356 while (block_entry !=
nullptr) {
357 BlockEntry *prev_entry = block_entry->
prev_ptr;
358 const BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
359 if (is_block_suitable_for_request(user_context, block, request)) {
360 #ifdef DEBUG_RUNTIME_INTERNAL 361 debug(user_context) <<
"BlockAllocator: found suitable block (" 362 <<
"user_context=" << (
void *)(user_context) <<
" " 363 <<
"block_resource=" << (
void *)block <<
" " 364 <<
"block_size=" << (
uint32_t)block->memory.size <<
" " 365 <<
"block_reserved=" << (
uint32_t)block->reserved <<
" " 366 <<
"request_size=" << (
uint32_t)request.size <<
" " 367 <<
"request_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 374 block_entry = prev_entry;
377 if (block_entry ==
nullptr) {
378 #ifdef DEBUG_RUNTIME_INTERNAL 379 debug(user_context) <<
"BlockAllocator: couldn't find suitable block! (" 380 <<
"user_context=" << (
void *)(user_context) <<
" " 381 <<
"request_size=" << (
uint32_t)request.size <<
" " 382 <<
"request_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 391 BlockAllocator::BlockEntry *
392 BlockAllocator::reserve_block_entry(
void *user_context,
const MemoryRequest &request) {
393 #ifdef DEBUG_RUNTIME_INTERNAL 394 debug(user_context) <<
"BlockAllocator: reserving block ... ! (" 395 <<
"requested_size=" << (
uint32_t)request.size <<
" " 396 <<
"requested_is_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 401 BlockEntry *block_entry = find_block_entry(user_context, request);
402 if (block_entry ==
nullptr) {
403 #ifdef DEBUG_RUNTIME_INTERNAL 404 debug(user_context) <<
"BlockAllocator: creating block ... ! (" 405 <<
"requested_size=" << (
uint32_t)request.size <<
" " 406 <<
"requested_is_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 411 block_entry = create_block_entry(user_context, request);
415 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
416 if (block->allocator ==
nullptr) {
417 block->allocator = create_region_allocator(user_context, block);
424 BlockAllocator::create_region_allocator(
void *user_context, BlockResource *block) {
425 #ifdef DEBUG_RUNTIME_INTERNAL 426 debug(user_context) <<
"BlockAllocator: Creating region allocator (" 427 <<
"user_context=" << (
void *)(user_context) <<
" " 428 <<
"block_resource=" << (
void *)(block) <<
")...";
432 user_context, block, {allocators.
system, allocators.
region});
434 if (region_allocator ==
nullptr) {
435 error(user_context) <<
"BlockAllocator: Failed to create new region allocator\n";
439 return region_allocator;
442 int BlockAllocator::destroy_region_allocator(
void *user_context, RegionAllocator *region_allocator) {
443 #ifdef DEBUG_RUNTIME_INTERNAL 444 debug(user_context) <<
"BlockAllocator: Destroying region allocator (" 445 <<
"user_context=" << (
void *)(user_context) <<
" " 446 <<
"region_allocator=" << (
void *)(region_allocator) <<
")...";
448 if (region_allocator ==
nullptr) {
454 BlockAllocator::BlockEntry *
455 BlockAllocator::create_block_entry(
void *user_context,
const MemoryRequest &request) {
457 error(user_context) <<
"BlockAllocator: No free blocks found! Maximum pool size reached (" 464 error(user_context) <<
"BlockAllocator: No free blocks found! Maximum block count reached (" 469 BlockEntry *block_entry = block_list.
append(user_context);
470 if (block_entry ==
nullptr) {
471 debug(user_context) <<
"BlockAllocator: Failed to allocate new block entry\n";
475 #ifdef DEBUG_RUNTIME_INTERNAL 476 debug(user_context) <<
"BlockAllocator: Creating block entry (" 477 <<
"block_entry=" << (
void *)(block_entry) <<
" " 478 <<
"block=" << (
void *)(block_entry->value) <<
" " 479 <<
"allocator=" << (
void *)(allocators.
block.
allocate) <<
")...";
483 MemoryRequest block_request = request;
484 conform(user_context, &block_request);
487 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
488 block->memory.size = block_request.size;
489 block->memory.handle =
nullptr;
490 block->memory.properties = block_request.properties;
491 block->memory.dedicated = block_request.dedicated;
493 block->allocator = create_region_allocator(user_context, block);
494 alloc_memory_block(user_context, block);
498 int BlockAllocator::release_block_entry(
void *user_context, BlockAllocator::BlockEntry *block_entry) {
499 #ifdef DEBUG_RUNTIME_INTERNAL 500 debug(user_context) <<
"BlockAllocator: Releasing block entry (" 501 <<
"block_entry=" << (
void *)(block_entry) <<
" " 502 <<
"block=" << (
void *)(block_entry->value) <<
")...";
504 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
505 if (block->allocator) {
506 return block->allocator->release(user_context);
511 int BlockAllocator::destroy_block_entry(
void *user_context, BlockAllocator::BlockEntry *block_entry) {
512 #ifdef DEBUG_RUNTIME_INTERNAL 513 debug(user_context) <<
"BlockAllocator: Destroying block entry (" 514 <<
"block_entry=" << (
void *)(block_entry) <<
" " 515 <<
"block=" << (
void *)(block_entry->value) <<
" " 518 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
519 if (block->allocator) {
520 destroy_region_allocator(user_context, block->allocator);
521 block->allocator =
nullptr;
523 free_memory_block(user_context, block);
524 block_list.
remove(user_context, block_entry);
528 int BlockAllocator::alloc_memory_block(
void *user_context, BlockResource *block) {
529 #ifdef DEBUG_RUNTIME_INTERNAL 530 debug(user_context) <<
"BlockAllocator: Allocating block (ptr=" << (
void *)block <<
" allocator=" << (
void *)allocators.
block.
allocate <<
")...";
533 MemoryBlock *memory_block = &(block->memory);
539 int BlockAllocator::free_memory_block(
void *user_context, BlockResource *block) {
540 #ifdef DEBUG_RUNTIME_INTERNAL 541 debug(user_context) <<
"BlockAllocator: Deallocating block (ptr=" << (
void *)block <<
" allocator=" << (
void *)allocators.
block.
deallocate <<
")...";
544 MemoryBlock *memory_block = &(block->memory);
546 memory_block->handle =
nullptr;
548 block->memory.size = 0;
552 size_t BlockAllocator::constrain_requested_size(
size_t size)
const {
553 size_t actual_size = size;
577 request->
size = (((request->
size + nm - 1) / nm) * nm);
634 return block_list.
size();
638 size_t total_size = 0;
640 for (block_entry = block_list.
front(); block_entry !=
nullptr; block_entry = block_entry->
next_ptr) {
642 if (block !=
nullptr) {
655 #endif // HALIDE_RUNTIME_BLOCK_ALLOCATOR_H size_t minimum_block_size
static int destroy(void *user_context, RegionAllocator *region_allocator)
Expr max(const FuncRef &a, const FuncRef &b)
Explicit overloads of min and max for FuncRef.
MemoryVisibility visibility
const MemoryAllocators & current_allocators() const
There is a bug in the Halide compiler.
MemoryProperties properties
bool collect(void *user_context)
void destroy(void *user_context)
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *memory_region)
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
DeallocateSystemFn deallocate
int release(void *user_context, MemoryRegion *memory_region)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline, and contains methods to using Halide's bounds tools to query properties of it.
RegionAllocator * allocator
WEAK const char * halide_memory_usage_name(MemoryUsage value)
int retain(void *user_context, MemoryRegion *region)
#define halide_abort_if_false(user_context, cond)
int reclaim(void *user_context, MemoryRegion *region)
EntryType * append(void *user_context)
const Config & default_config() const
unsigned __INT32_TYPE__ uint32_t
SystemMemoryAllocatorFns system
const Config & current_config() const
size_t maximum_block_size
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
Not visible externally, similar to 'static' linkage in C.
int conform(void *user_context, MemoryRequest *request) const
MemoryProperties properties
DeallocateBlockFn deallocate
AllocateSystemFn allocate
MemoryRegionAllocatorFns region
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
bool collect(void *user_context)
size_t block_count() const
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
void initialize(void *user_context, uint32_t entry_size, uint32_t capacity=default_capacity, const SystemMemoryAllocatorFns &allocator=default_allocator())
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
BlockAllocator & operator=(const BlockAllocator &)=delete
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
ConformBlockRequestFn conform
unsigned __INT64_TYPE__ uint64_t
MemoryBlockAllocatorFns block
WEAK const char * halide_memory_caching_name(MemoryCaching value)
static void destroy(void *user_context, BlockAllocator *block_allocator)
int reclaim(void *user_context, MemoryRegion *memory_region)
signed __INT32_TYPE__ int32_t
void remove(void *user_context, EntryType *entry_ptr)
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
size_t maximum_block_count