1 #ifndef HALIDE_RUNTIME_REGION_ALLOCATOR_H 2 #define HALIDE_RUNTIME_REGION_ALLOCATOR_H 4 #include "../HalideRuntime.h" 5 #include "../printer.h" 53 bool collect(
void *user_context);
54 int release(
void *user_context);
55 int destroy(
void *user_context);
86 int destroy_block_region(
void *user_context,
BlockRegion *region);
89 int alloc_block_region(
void *user_context,
BlockRegion *region);
92 int release_block_region(
void *user_context,
BlockRegion *region);
95 int free_block_region(
void *user_context,
BlockRegion *region);
98 bool is_last_block_region(
void *user_context,
const BlockRegion *region)
const;
104 bool is_block_region_suitable_for_request(
void *user_context,
const BlockRegion *region,
const MemoryRequest &request)
const;
107 size_t region_count(
void *user_context)
const;
119 if (result ==
nullptr) {
130 instance->
destroy(user_context);
136 int RegionAllocator::initialize(
void *user_context,
BlockResource *mb,
const MemoryAllocators &ma) {
141 MemoryRequest block_request = {};
143 block_request.offset = 0;
148 block->
regions = create_block_region(user_context, block_request);
160 request->
offset = actual_offset;
161 request->
size = actual_size;
171 int error_code =
conform(user_context, ®ion_request);
173 #ifdef DEBUG_RUNTIME_INTERNAL 174 debug(user_context) <<
"RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
180 if (remaining < region_request.
size) {
181 #ifdef DEBUG_RUNTIME_INTERNAL 182 debug(user_context) <<
"RegionAllocator: Unable to reserve more memory from block " 183 <<
"-- requested size (" << (
int32_t)(region_request.
size) <<
" bytes) " 184 <<
"greater than available (" << (
int32_t)(remaining) <<
" bytes)";
189 BlockRegion *block_region = find_block_region(user_context, region_request);
190 if (block_region ==
nullptr) {
191 #ifdef DEBUG_RUNTIME_INTERNAL 192 debug(user_context) <<
"RegionAllocator: Failed to locate region for requested size (" 198 if (can_split(block_region, region_request)) {
199 #ifdef DEBUG_RUNTIME_INTERNAL 200 debug(user_context) <<
"RegionAllocator: Splitting region of size ( " << (
int32_t)(block_region->
memory.
size) <<
") " 201 <<
"to accomodate requested size (" << (
int32_t)(region_request.
size) <<
" bytes)";
203 split_block_region(user_context, block_region, region_request);
206 alloc_block_region(user_context, block_region);
217 return release_block_region(user_context, block_region);
227 release_block_region(user_context, block_region);
228 free_block_region(user_context, block_region);
242 if (block_region ==
nullptr) {
245 if (block_region->
block_ptr ==
nullptr) {
251 bool RegionAllocator::is_last_block_region(
void *user_context,
const BlockRegion *region)
const {
252 return ((region ==
nullptr) || (region == region->
next_ptr) || (region->
next_ptr ==
nullptr));
255 bool RegionAllocator::is_block_region_suitable_for_request(
void *user_context,
const BlockRegion *region,
const MemoryRequest &request)
const {
256 if (!is_available(region)) {
257 #ifdef DEBUG_RUNTIME_INTERNAL 258 debug(user_context) <<
" skipping block region ... not available! (" 259 <<
" block_region=" << (
void *)region
260 <<
" region_size=" << (
uint32_t)(region->memory.size)
266 MemoryRequest region_request = request;
267 int error_code =
conform(user_context, ®ion_request);
269 #ifdef DEBUG_RUNTIME_INTERNAL 270 debug(user_context) <<
"RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
276 if (!is_compatible_block_region(region, region_request.properties)) {
277 #ifdef DEBUG_RUNTIME_INTERNAL 278 debug(user_context) <<
" skipping block region ... incompatible properties! (" 279 <<
" block_region=" << (
void *)region
280 <<
" region_size=" << (
uint32_t)(region->memory.size)
287 if (region_request.size > region->memory.size) {
288 #ifdef DEBUG_RUNTIME_INTERNAL 289 debug(user_context) <<
" skipping block region ... not enough space for adjusted size! (" 290 <<
" block_region=" << (
void *)region
291 <<
" request_size=" << (
uint32_t)(request.size)
292 <<
" actual_size=" << (
uint32_t)(region_request.size)
293 <<
" region_size=" << (
uint32_t)(region->memory.size)
301 #ifdef DEBUG_RUNTIME_INTERNAL 302 debug(user_context) <<
" found suitable block region! (" 303 <<
" block_region=" << (
void *)region
304 <<
" request_size=" << (
uint32_t)(request.size)
305 <<
" actual_size=" << (
uint32_t)(region_request.size)
306 <<
" region_size=" << (
uint32_t)(region->memory.size)
315 BlockRegion *RegionAllocator::find_block_region(
void *user_context,
const MemoryRequest &request) {
316 #ifdef DEBUG_RUNTIME_INTERNAL 317 debug(user_context) <<
"RegionAllocator: find block region ( " 318 <<
"user_context=" << (
void *)(user_context) <<
" " 319 <<
"requested_size=" << (
uint32_t)request.size <<
" " 320 <<
"requested_is_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 325 BlockRegion *block_region = block->
regions;
326 while (block_region !=
nullptr) {
327 if (is_block_region_suitable_for_request(user_context, block_region, request)) {
328 #ifdef DEBUG_RUNTIME_INTERNAL 329 debug(user_context) <<
"RegionAllocator: found suitable region ( " 330 <<
"user_context=" << (
void *)(user_context) <<
" " 331 <<
"block_resource=" << (
void *)block <<
" " 334 <<
"requested_size=" << (
uint32_t)request.size <<
" " 335 <<
"requested_is_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 343 if (is_last_block_region(user_context, block_region)) {
344 block_region =
nullptr;
347 block_region = block_region->next_ptr;
350 if (block_region ==
nullptr) {
351 #ifdef DEBUG_RUNTIME_INTERNAL 352 debug(user_context) <<
"RegionAllocator: couldn't find suitable region! (" 353 <<
"user_context=" << (
void *)(user_context) <<
" " 354 <<
"requested_size=" << (
uint32_t)request.size <<
" " 355 <<
"requested_is_dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 365 bool RegionAllocator::is_available(
const BlockRegion *block_region)
const {
366 if (block_region ==
nullptr) {
369 if (block_region->usage_count > 0) {
378 bool RegionAllocator::can_coalesce(
const BlockRegion *block_region)
const {
379 if (!is_available(block_region)) {
382 if (is_available(block_region->prev_ptr)) {
385 if (is_available(block_region->next_ptr)) {
391 BlockRegion *RegionAllocator::coalesce_block_regions(
void *user_context, BlockRegion *block_region) {
393 if ((block_region->usage_count == 0) && (block_region->memory.handle !=
nullptr)) {
394 #ifdef DEBUG_RUNTIME_INTERNAL 395 debug(user_context) <<
"RegionAllocator: Freeing unused region to coalesce (" 396 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 397 <<
"block_region=" << (
void *)block_region <<
" " 398 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 403 MemoryRegion *memory_region = &(block_region->memory);
405 block_region->memory.handle =
nullptr;
408 BlockRegion *prev_region = block_region->prev_ptr;
409 if (is_available(prev_region) && (prev_region != block_region)) {
411 #ifdef DEBUG_RUNTIME_INTERNAL 412 debug(user_context) <<
"RegionAllocator: Coalescing " 413 <<
"previous region (offset=" << (
int32_t)prev_region->memory.offset <<
" size=" << (
int32_t)(prev_region->memory.size) <<
" bytes) " 414 <<
"into current region (offset=" << (
int32_t)block_region->memory.offset <<
" size=" << (
int32_t)(block_region->memory.size) <<
" bytes)!";
417 prev_region->next_ptr = block_region->next_ptr;
418 if (block_region->next_ptr) {
419 block_region->next_ptr->prev_ptr = prev_region;
421 prev_region->memory.size += block_region->memory.size;
422 destroy_block_region(user_context, block_region);
423 block_region = prev_region;
426 BlockRegion *next_region = block_region->next_ptr;
427 if (is_available(next_region) && (next_region != block_region)) {
429 #ifdef DEBUG_RUNTIME_INTERNAL 430 debug(user_context) <<
"RegionAllocator: Coalescing " 431 <<
"next region (offset=" << (
int32_t)next_region->memory.offset <<
" size=" << (
int32_t)(next_region->memory.size) <<
" bytes) " 432 <<
"into current region (offset=" << (
int32_t)block_region->memory.offset <<
" size=" << (
int32_t)(block_region->memory.size) <<
" bytes)";
435 if (next_region->next_ptr) {
436 next_region->next_ptr->prev_ptr = block_region;
438 block_region->next_ptr = next_region->next_ptr;
439 block_region->memory.size += next_region->memory.size;
440 destroy_block_region(user_context, next_region);
446 bool RegionAllocator::can_split(
const BlockRegion *block_region,
const MemoryRequest &split_request)
const {
447 return (block_region && (block_region->memory.size > split_request.size) && (block_region->usage_count == 0));
450 BlockRegion *RegionAllocator::split_block_region(
void *user_context, BlockRegion *block_region,
const MemoryRequest &request) {
452 if ((block_region->usage_count == 0) && (block_region->memory.handle !=
nullptr)) {
453 #ifdef DEBUG_RUNTIME_INTERNAL 454 debug(user_context) <<
"RegionAllocator: Split deallocate region (" 455 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 456 <<
"block_region=" << (
void *)block_region <<
" " 457 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 458 <<
"block_reserved=" << (
uint32_t)block_region->block_ptr->reserved <<
" " 462 MemoryRegion *memory_region = &(block_region->memory);
464 block_region->memory.handle =
nullptr;
467 MemoryRequest split_request = request;
468 split_request.size = block_region->memory.size - request.size;
469 split_request.offset = block_region->memory.offset + request.size;
471 #ifdef DEBUG_RUNTIME_INTERNAL 472 debug(user_context) <<
"RegionAllocator: Splitting " 473 <<
"current region (offset=" << (
int32_t)block_region->memory.offset <<
" size=" << (
int32_t)(block_region->memory.size) <<
" bytes) " 474 <<
"to create empty region (offset=" << (
int32_t)split_request.offset <<
" size=" << (
int32_t)(split_request.size) <<
" bytes)";
476 BlockRegion *next_region = block_region->next_ptr;
477 BlockRegion *empty_region = create_block_region(user_context, split_request);
480 empty_region->next_ptr = next_region;
482 next_region->prev_ptr = empty_region;
484 empty_region->prev_ptr = block_region;
485 block_region->next_ptr = empty_region;
486 block_region->memory.size -= empty_region->memory.size;
490 BlockRegion *RegionAllocator::create_block_region(
void *user_context,
const MemoryRequest &request) {
491 #ifdef DEBUG_RUNTIME_INTERNAL 492 debug(user_context) <<
"RegionAllocator: Creating block region request (" 493 <<
"user_context=" << (
void *)(user_context) <<
" " 494 <<
"offset=" << (
uint32_t)request.offset <<
" " 495 <<
"size=" << (
uint32_t)request.size <<
" " 496 <<
"alignment=" << (
uint32_t)request.properties.alignment <<
" " 497 <<
"dedicated=" << (request.dedicated ?
"true" :
"false") <<
" " 503 MemoryRequest region_request = request;
504 int error_code =
conform(user_context, ®ion_request);
506 #ifdef DEBUG_RUNTIME_INTERNAL 507 debug(user_context) <<
"RegionAllocator: Failed to conform request for new block region!\n";
512 if (region_request.size == 0) {
513 #ifdef DEBUG_RUNTIME_INTERNAL 514 debug(user_context) <<
"RegionAllocator: Failed to allocate new block region ... region size was zero!\n";
519 BlockRegion *block_region =
static_cast<BlockRegion *
>(arena->
reserve(user_context,
true));
520 if (block_region ==
nullptr) {
521 #ifdef DEBUG_RUNTIME_INTERNAL 522 debug(user_context) <<
"RegionAllocator: Failed to allocate new block region!\n";
527 block_region->memory.handle =
nullptr;
528 block_region->memory.offset = region_request.offset;
529 block_region->memory.size = region_request.size;
530 block_region->memory.properties = region_request.properties;
531 block_region->memory.dedicated = region_request.dedicated;
533 block_region->block_ptr = block;
534 block_region->usage_count = 0;
536 #ifdef DEBUG_RUNTIME_INTERNAL 537 debug(user_context) <<
"RegionAllocator: Created block region allocation (" 538 <<
"user_context=" << (
void *)(user_context) <<
" " 539 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 540 <<
"block_region=" << (
void *)block_region <<
" " 541 <<
"memory_offset=" << (
uint32_t)(block_region->memory.offset) <<
" " 542 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 549 int RegionAllocator::release_block_region(
void *user_context, BlockRegion *block_region) {
550 #ifdef DEBUG_RUNTIME_INTERNAL 551 debug(user_context) <<
"RegionAllocator: Releasing block region (" 552 <<
"user_context=" << (
void *)(user_context) <<
" " 553 <<
"block_ptr=" << ((block_region) ? ((
void *)block_region->block_ptr) :
nullptr) <<
" " 554 <<
"block_region=" << (
void *)block_region <<
" " 555 <<
"usage_count=" << ((block_region) ? (
uint32_t)(block_region->usage_count) : 0) <<
" " 556 <<
"memory_offset=" << ((block_region) ? (
uint32_t)(block_region->memory.offset) : 0) <<
" " 557 <<
"memory_size=" << ((block_region) ? (
uint32_t)(block_region->memory.size) : 0) <<
" " 560 if (block_region ==
nullptr) {
564 if (block_region->usage_count > 0) {
570 #ifdef DEBUG_RUNTIME_INTERNAL 571 debug(user_context) <<
" releasing region (" 572 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 573 <<
"block_region=" << (
void *)block_region <<
" " 574 <<
"memory_offset=" << (
uint32_t)(block_region->memory.offset) <<
" " 575 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 576 <<
"block_reserved=" << (
uint32_t)(block->
reserved - block_region->memory.size) <<
" " 580 block->
reserved -= block_region->memory.size;
586 int RegionAllocator::destroy_block_region(
void *user_context, BlockRegion *block_region) {
587 #ifdef DEBUG_RUNTIME_INTERNAL 588 debug(user_context) <<
"RegionAllocator: Destroying block region (" 589 <<
"user_context=" << (
void *)(user_context) <<
" " 590 <<
"block_region=" << (
void *)(block_region) <<
") ...";
593 block_region->usage_count = 0;
594 release_block_region(user_context, block_region);
595 free_block_region(user_context, block_region);
596 arena->
reclaim(user_context, block_region);
600 int RegionAllocator::alloc_block_region(
void *user_context, BlockRegion *block_region) {
601 #ifdef DEBUG_RUNTIME_INTERNAL 602 debug(user_context) <<
"RegionAllocator: Allocating region (user_context=" << (
void *)(user_context)
603 <<
" size=" << (
int32_t)(block_region->memory.size)
604 <<
" offset=" << (
int32_t)block_region->memory.offset <<
")";
609 MemoryRegion *memory_region = &(block_region->memory);
610 if (memory_region->handle ==
nullptr) {
611 error_code = allocators.
region.
allocate(user_context, memory_region);
612 memory_region->is_owner =
true;
614 #ifdef DEBUG_RUNTIME_INTERNAL 615 debug(user_context) <<
" allocating region (" 616 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 617 <<
"block_region=" << (
void *)block_region <<
" " 618 <<
"memory_offset=" << (
uint32_t)(block_region->memory.offset) <<
" " 619 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 626 #ifdef DEBUG_RUNTIME_INTERNAL 627 debug(user_context) <<
" re-using region (" 628 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 629 <<
"block_region=" << (
void *)block_region <<
" " 630 <<
"memory_offset=" << (
uint32_t)(block_region->memory.offset) <<
" " 631 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 637 block->
reserved += block_region->memory.size;
641 int RegionAllocator::free_block_region(
void *user_context, BlockRegion *block_region) {
642 #ifdef DEBUG_RUNTIME_INTERNAL 643 debug(user_context) <<
"RegionAllocator: Freeing block region (" 644 <<
"user_context=" << (
void *)(user_context) <<
" " 645 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 646 <<
"block_region=" << (
void *)(block_region) <<
" " 647 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 648 <<
"status=" << (
uint32_t)block_region->status <<
" " 649 <<
"usage_count=" << (
uint32_t)block_region->usage_count <<
" " 652 if ((block_region->usage_count == 0) && (block_region->memory.handle !=
nullptr)) {
653 #ifdef DEBUG_RUNTIME_INTERNAL 654 debug(user_context) <<
" deallocating region (" 655 <<
"block_ptr=" << (
void *)block_region->block_ptr <<
" " 656 <<
"block_region=" << (
void *)block_region <<
" " 657 <<
"memory_size=" << (
uint32_t)(block_region->memory.size) <<
" " 663 MemoryRegion *memory_region = &(block_region->memory);
665 block_region->memory.handle =
nullptr;
667 block_region->usage_count = 0;
673 #ifdef DEBUG_RUNTIME_INTERNAL 674 debug(user_context) <<
"RegionAllocator: Releasing all regions (" 675 <<
"user_context=" << (
void *)(user_context) <<
") ...";
679 while (block_region !=
nullptr) {
680 release_block_region(user_context, block_region);
681 if (is_last_block_region(user_context, block_region)) {
684 block_region = block_region->
next_ptr;
690 #ifdef DEBUG_RUNTIME_INTERNAL 691 debug(user_context) <<
"RegionAllocator: Collecting free block regions (" 692 <<
"user_context=" << (
void *)(user_context) <<
") ...";
699 debug(user_context) <<
" collecting unused regions (" 700 <<
"block_ptr=" << (
void *)block <<
" " 705 bool has_collected =
false;
707 while (block_region !=
nullptr) {
708 #ifdef DEBUG_RUNTIME_INTERNAL 710 debug(user_context) <<
" checking region (" 711 <<
"block_ptr=" << (
void *)block_region->
block_ptr <<
" " 712 <<
"block_region=" << (
void *)block_region <<
" " 720 if (can_coalesce(block_region)) {
721 #ifdef DEBUG_RUNTIME_INTERNAL 723 debug(user_context) <<
" collecting region (" 724 <<
"block_ptr=" << (
void *)block_region->
block_ptr <<
" " 725 <<
"block_region=" << (
void *)block_region <<
" " 730 block_region = coalesce_block_regions(user_context, block_region);
731 has_collected =
true;
733 #ifdef DEBUG_RUNTIME_INTERNAL 737 #ifdef DEBUG_RUNTIME_INTERNAL 738 available_bytes += is_available(block_region) ? block_region->
memory.
size : 0;
740 if (is_last_block_region(user_context, block_region)) {
743 block_region = block_region->
next_ptr;
745 #ifdef DEBUG_RUNTIME_INTERNAL 746 debug(user_context) <<
" scanned active regions (" 747 <<
"block_ptr=" << (
void *)block <<
" " 748 <<
"total_count=" << (
uint32_t)(collected_count + remaining_count) <<
" " 750 <<
"scanned_bytes=" << (
uint32_t)(scanned_bytes) <<
" " 751 <<
"available_bytes=" << (
uint32_t)(available_bytes) <<
" " 756 #ifdef DEBUG_RUNTIME_INTERNAL 757 debug(user_context) <<
" collected unused regions (" 758 <<
"block_ptr=" << (
void *)block <<
" " 759 <<
"collected_count=" << (
uint32_t)collected_count <<
" " 760 <<
"remaining_count=" << (
uint32_t)remaining_count <<
" " 765 return has_collected;
769 #ifdef DEBUG_RUNTIME_INTERNAL 770 debug(user_context) <<
"RegionAllocator: Destroying all block regions (" 771 <<
"user_context=" << (
void *)(user_context) <<
") ...";
773 if (block->
regions !=
nullptr) {
776 if (is_last_block_region(user_context, block_region)) {
777 destroy_block_region(user_context, block_region);
778 block_region =
nullptr;
781 block_region = block_region->
next_ptr;
782 destroy_block_region(user_context, prev_region);
789 if (arena !=
nullptr) {
818 size_t RegionAllocator::region_count(
void *user_context)
const {
819 if (block ==
nullptr) {
823 for (BlockRegion
const *region = block->
regions; !is_last_block_region(user_context, region); region = region->next_ptr) {
839 #endif // HALIDE_RUNTIME_REGION_ALLOCATOR_H
static int destroy(void *user_context, RegionAllocator *region_allocator)
MemoryVisibility visibility
SystemMemoryAllocatorFns system
MemoryProperties properties
ConformBlockRegionFn conform
MemoryProperties properties
static MemoryArena * create(void *user_context, const Config &config, const SystemMemoryAllocatorFns &allocator=default_allocator())
int retain(void *user_context, MemoryRegion *memory_region)
static void destroy(void *user_context, MemoryArena *arena)
~RegionAllocator()=delete
void reclaim(void *user_context, void *ptr)
RegionAllocator & operator=(const RegionAllocator &)=delete
BlockResource * block_ptr
DeallocateRegionFn deallocate
DeallocateSystemFn deallocate
int release(void *user_context, MemoryRegion *memory_region)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline, and contains methods to using Halide's bounds tools to query properties of it.
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
BlockResource * block_resource() const
RegionAllocator * allocator
WEAK const char * halide_memory_usage_name(MemoryUsage value)
MemoryRegionAllocatorFns region
#define halide_abort_if_false(user_context, cond)
int conform(void *user_context, MemoryRequest *request) const
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
unsigned __INT32_TYPE__ uint32_t
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
Not visible externally, similar to 'static' linkage in C.
AllocateSystemFn allocate
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
bool collect(void *user_context)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
static constexpr uint32_t default_capacity
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
unsigned __INT64_TYPE__ uint64_t
AllocateRegionFn allocate
WEAK const char * halide_memory_caching_name(MemoryCaching value)
int reclaim(void *user_context, MemoryRegion *memory_region)
signed __INT32_TYPE__ int32_t
void * reserve(void *user_context, bool initialize=false)
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)