Halide  17.0.2
Halide compiler and libraries
region_allocator.h
Go to the documentation of this file.
1 #ifndef HALIDE_RUNTIME_REGION_ALLOCATOR_H
2 #define HALIDE_RUNTIME_REGION_ALLOCATOR_H
3 
4 #include "../HalideRuntime.h"
5 #include "../printer.h"
6 #include "memory_arena.h"
7 #include "memory_resources.h"
8 
9 namespace Halide {
10 namespace Runtime {
11 namespace Internal {
12 
13 // --
14 
15 /** Allocator class interface for sub-allocating a contiguous
16  * memory block into smaller regions of memory. This class only
17  * manages the address creation for the regions -- allocation
18  * callback functions are used to request the memory from the
19  * necessary system or API calls. This class is intended to be
20  * used inside of a higher level memory management class that
21  * provides thread safety, policy management and API
22  * integration for a specific runtime API (eg Vulkan, OpenCL, etc)
23  */
25 public:
26  // disable copy constructors and assignment
27  RegionAllocator(const RegionAllocator &) = delete;
28  RegionAllocator &operator=(const RegionAllocator &) = delete;
29 
30  // disable non-factory based construction
31  RegionAllocator() = delete;
32  ~RegionAllocator() = delete;
33 
34  // Allocators for the different types of memory we need to allocate
38  };
39 
40  // Factory methods for creation / destruction
41  static RegionAllocator *create(void *user_context, BlockResource *block, const MemoryAllocators &ma);
42  static int destroy(void *user_context, RegionAllocator *region_allocator);
43 
44  // Returns the allocator class instance for the given allocation (or nullptr)
45  static RegionAllocator *find_allocator(void *user_context, MemoryRegion *memory_region);
46 
47  // Public interface methods
48  MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
49  int conform(void *user_context, MemoryRequest *request) const; //< conform the given request into a suitable allocation
50  int release(void *user_context, MemoryRegion *memory_region); //< unmark and cache the region for reuse
51  int reclaim(void *user_context, MemoryRegion *memory_region); //< free the region and consolidate
52  int retain(void *user_context, MemoryRegion *memory_region); //< retain the region and increase usage count
53  bool collect(void *user_context); //< returns true if any blocks were removed
54  int release(void *user_context);
55  int destroy(void *user_context);
56 
57  // Returns the currently managed block resource
59 
60 private:
61  // Initializes a new instance
62  int initialize(void *user_context, BlockResource *block, const MemoryAllocators &ma);
63 
64  // Search through allocated block regions (Best-Fit)
65  BlockRegion *find_block_region(void *user_context, const MemoryRequest &request);
66 
67  // Returns true if block region is unused and available
68  bool is_available(const BlockRegion *region) const;
69 
70  // Returns true if neighbouring block regions to the given region can be coalesced into one
71  bool can_coalesce(const BlockRegion *region) const;
72 
73  // Merges available neighbouring block regions into the given region
74  BlockRegion *coalesce_block_regions(void *user_context, BlockRegion *region);
75 
76  // Returns true if the given region can be split to accomodate the given size
77  bool can_split(const BlockRegion *region, const MemoryRequest &request) const;
78 
79  // Splits the given block region into a smaller region to accomodate the given size, followed by empty space for the remaining
80  BlockRegion *split_block_region(void *user_context, BlockRegion *region, const MemoryRequest &request);
81 
82  // Creates a new block region and adds it to the region list
83  BlockRegion *create_block_region(void *user_context, const MemoryRequest &request);
84 
85  // Creates a new block region and adds it to the region list
86  int destroy_block_region(void *user_context, BlockRegion *region);
87 
88  // Invokes the allocation callback to allocate memory for the block region
89  int alloc_block_region(void *user_context, BlockRegion *region);
90 
91  // Releases a block region and leaves it in the list for further allocations
92  int release_block_region(void *user_context, BlockRegion *region);
93 
94  // Invokes the deallocation callback to free memory for the block region
95  int free_block_region(void *user_context, BlockRegion *region);
96 
97  // Returns true if the given block region is the last region in the list
98  bool is_last_block_region(void *user_context, const BlockRegion *region) const;
99 
100  // Returns true if the given block region is compatible with the given properties
101  bool is_compatible_block_region(const BlockRegion *region, const MemoryProperties &properties) const;
102 
103  // Returns true if the given block region is suitable for the requested allocation
104  bool is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const;
105 
106  // Returns the number of active regions for the block;
107  size_t region_count(void *user_context) const;
108 
109  BlockResource *block = nullptr;
110  MemoryArena *arena = nullptr;
111  MemoryAllocators allocators;
112 };
113 
114 RegionAllocator *RegionAllocator::create(void *user_context, BlockResource *block_resource, const MemoryAllocators &allocators) {
115  halide_abort_if_false(user_context, allocators.system.allocate != nullptr);
116  RegionAllocator *result = reinterpret_cast<RegionAllocator *>(
117  allocators.system.allocate(user_context, sizeof(RegionAllocator)));
118 
119  if (result == nullptr) {
120  return nullptr;
121  }
122 
123  result->initialize(user_context, block_resource, allocators);
124  return result;
125 }
126 
127 int RegionAllocator::destroy(void *user_context, RegionAllocator *instance) {
128  halide_abort_if_false(user_context, instance != nullptr);
129  const MemoryAllocators &allocators = instance->allocators;
130  instance->destroy(user_context);
131  halide_abort_if_false(user_context, allocators.system.deallocate != nullptr);
132  allocators.system.deallocate(user_context, instance);
133  return 0;
134 }
135 
136 int RegionAllocator::initialize(void *user_context, BlockResource *mb, const MemoryAllocators &ma) {
137  block = mb;
138  allocators = ma;
139  arena = MemoryArena::create(user_context, {sizeof(BlockRegion), MemoryArena::default_capacity, 0}, allocators.system);
140  halide_abort_if_false(user_context, arena != nullptr);
141  MemoryRequest block_request = {};
142  block_request.size = block->memory.size;
143  block_request.offset = 0;
144  block_request.alignment = block->memory.properties.alignment;
145  block_request.properties = block->memory.properties;
146  block_request.dedicated = block->memory.dedicated;
147  block->allocator = this;
148  block->regions = create_block_region(user_context, block_request);
149  return 0;
150 }
151 
152 int RegionAllocator::conform(void *user_context, MemoryRequest *request) const {
153  if (allocators.region.conform) {
154  return allocators.region.conform(user_context, request);
155  } else {
156  size_t actual_alignment = conform_alignment(request->alignment, block->memory.properties.alignment);
157  size_t actual_offset = aligned_offset(request->offset, actual_alignment);
158  size_t actual_size = conform_size(actual_offset, request->size, actual_alignment, block->memory.properties.nearest_multiple);
159  request->alignment = actual_alignment;
160  request->offset = actual_offset;
161  request->size = actual_size;
162  }
163  return 0;
164 }
165 
166 MemoryRegion *RegionAllocator::reserve(void *user_context, const MemoryRequest &request) {
167  halide_abort_if_false(user_context, request.size > 0);
168 
169  MemoryRequest region_request = request;
170 
171  int error_code = conform(user_context, &region_request);
172  if (error_code) {
173 #ifdef DEBUG_RUNTIME_INTERNAL
174  debug(user_context) << "RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
175 #endif
176  return nullptr;
177  }
178 
179  size_t remaining = block->memory.size - block->reserved;
180  if (remaining < region_request.size) {
181 #ifdef DEBUG_RUNTIME_INTERNAL
182  debug(user_context) << "RegionAllocator: Unable to reserve more memory from block "
183  << "-- requested size (" << (int32_t)(region_request.size) << " bytes) "
184  << "greater than available (" << (int32_t)(remaining) << " bytes)";
185 #endif
186  return nullptr;
187  }
188 
189  BlockRegion *block_region = find_block_region(user_context, region_request);
190  if (block_region == nullptr) {
191 #ifdef DEBUG_RUNTIME_INTERNAL
192  debug(user_context) << "RegionAllocator: Failed to locate region for requested size ("
193  << (int32_t)(request.size) << " bytes)";
194 #endif
195  return nullptr;
196  }
197 
198  if (can_split(block_region, region_request)) {
199 #ifdef DEBUG_RUNTIME_INTERNAL
200  debug(user_context) << "RegionAllocator: Splitting region of size ( " << (int32_t)(block_region->memory.size) << ") "
201  << "to accomodate requested size (" << (int32_t)(region_request.size) << " bytes)";
202 #endif
203  split_block_region(user_context, block_region, region_request);
204  }
205 
206  alloc_block_region(user_context, block_region);
207  return reinterpret_cast<MemoryRegion *>(block_region);
208 }
209 
210 int RegionAllocator::release(void *user_context, MemoryRegion *memory_region) {
211  BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
212  halide_abort_if_false(user_context, block_region != nullptr);
213  halide_abort_if_false(user_context, block_region->block_ptr == block);
214  if (block_region->usage_count > 0) {
215  block_region->usage_count--;
216  }
217  return release_block_region(user_context, block_region);
218 }
219 
220 int RegionAllocator::reclaim(void *user_context, MemoryRegion *memory_region) {
221  BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
222  halide_abort_if_false(user_context, block_region != nullptr);
223  halide_abort_if_false(user_context, block_region->block_ptr == block);
224  if (block_region->usage_count > 0) {
225  block_region->usage_count--;
226  }
227  release_block_region(user_context, block_region);
228  free_block_region(user_context, block_region);
229  return 0;
230 }
231 
232 int RegionAllocator::retain(void *user_context, MemoryRegion *memory_region) {
233  BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
234  halide_abort_if_false(user_context, block_region != nullptr);
235  halide_abort_if_false(user_context, block_region->block_ptr == block);
236  block_region->usage_count++;
237  return 0;
238 }
239 
240 RegionAllocator *RegionAllocator::find_allocator(void *user_context, MemoryRegion *memory_region) {
241  BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
242  if (block_region == nullptr) {
243  return nullptr;
244  }
245  if (block_region->block_ptr == nullptr) {
246  return nullptr;
247  }
248  return block_region->block_ptr->allocator;
249 }
250 
251 bool RegionAllocator::is_last_block_region(void *user_context, const BlockRegion *region) const {
252  return ((region == nullptr) || (region == region->next_ptr) || (region->next_ptr == nullptr));
253 }
254 
255 bool RegionAllocator::is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const {
256  if (!is_available(region)) {
257 #ifdef DEBUG_RUNTIME_INTERNAL
258  debug(user_context) << " skipping block region ... not available! ("
259  << " block_region=" << (void *)region
260  << " region_size=" << (uint32_t)(region->memory.size)
261  << ")";
262 #endif
263  return false;
264  }
265 
266  MemoryRequest region_request = request;
267  int error_code = conform(user_context, &region_request);
268  if (error_code) {
269 #ifdef DEBUG_RUNTIME_INTERNAL
270  debug(user_context) << "RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
271 #endif
272  return false;
273  }
274 
275  // skip incompatible block regions for this request
276  if (!is_compatible_block_region(region, region_request.properties)) {
277 #ifdef DEBUG_RUNTIME_INTERNAL
278  debug(user_context) << " skipping block region ... incompatible properties! ("
279  << " block_region=" << (void *)region
280  << " region_size=" << (uint32_t)(region->memory.size)
281  << ")";
282 #endif
283  return false;
284  }
285 
286  // is the adjusted size larger than the current region?
287  if (region_request.size > region->memory.size) {
288 #ifdef DEBUG_RUNTIME_INTERNAL
289  debug(user_context) << " skipping block region ... not enough space for adjusted size! ("
290  << " block_region=" << (void *)region
291  << " request_size=" << (uint32_t)(request.size)
292  << " actual_size=" << (uint32_t)(region_request.size)
293  << " region_size=" << (uint32_t)(region->memory.size)
294  << ")";
295 #endif
296  return false;
297  }
298 
299  // will the adjusted size fit within the remaining unallocated space?
300  if ((region_request.size + block->reserved) <= block->memory.size) {
301 #ifdef DEBUG_RUNTIME_INTERNAL
302  debug(user_context) << " found suitable block region! ("
303  << " block_region=" << (void *)region
304  << " request_size=" << (uint32_t)(request.size)
305  << " actual_size=" << (uint32_t)(region_request.size)
306  << " region_size=" << (uint32_t)(region->memory.size)
307  << ")";
308 #endif
309  return true; // you betcha
310  }
311 
312  return false;
313 }
314 
315 BlockRegion *RegionAllocator::find_block_region(void *user_context, const MemoryRequest &request) {
316 #ifdef DEBUG_RUNTIME_INTERNAL
317  debug(user_context) << "RegionAllocator: find block region ( "
318  << "user_context=" << (void *)(user_context) << " "
319  << "requested_size=" << (uint32_t)request.size << " "
320  << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
321  << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
322  << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
323  << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
324 #endif
325  BlockRegion *block_region = block->regions;
326  while (block_region != nullptr) {
327  if (is_block_region_suitable_for_request(user_context, block_region, request)) {
328 #ifdef DEBUG_RUNTIME_INTERNAL
329  debug(user_context) << "RegionAllocator: found suitable region ( "
330  << "user_context=" << (void *)(user_context) << " "
331  << "block_resource=" << (void *)block << " "
332  << "block_size=" << (uint32_t)block->memory.size << " "
333  << "block_reserved=" << (uint32_t)block->reserved << " "
334  << "requested_size=" << (uint32_t)request.size << " "
335  << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
336  << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
337  << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
338  << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
339 #endif
340  return block_region;
341  }
342 
343  if (is_last_block_region(user_context, block_region)) {
344  block_region = nullptr; // end of list ... nothing found
345  break;
346  }
347  block_region = block_region->next_ptr;
348  }
349 
350  if (block_region == nullptr) {
351 #ifdef DEBUG_RUNTIME_INTERNAL
352  debug(user_context) << "RegionAllocator: couldn't find suitable region! ("
353  << "user_context=" << (void *)(user_context) << " "
354  << "requested_size=" << (uint32_t)request.size << " "
355  << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
356  << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
357  << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
358  << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
359 #endif
360  }
361 
362  return block_region;
363 }
364 
365 bool RegionAllocator::is_available(const BlockRegion *block_region) const {
366  if (block_region == nullptr) {
367  return false;
368  }
369  if (block_region->usage_count > 0) {
370  return false;
371  }
372  if (block_region->status != AllocationStatus::Available) {
373  return false;
374  }
375  return true;
376 }
377 
378 bool RegionAllocator::can_coalesce(const BlockRegion *block_region) const {
379  if (!is_available(block_region)) {
380  return false;
381  }
382  if (is_available(block_region->prev_ptr)) {
383  return true;
384  }
385  if (is_available(block_region->next_ptr)) {
386  return true;
387  }
388  return false;
389 }
390 
391 BlockRegion *RegionAllocator::coalesce_block_regions(void *user_context, BlockRegion *block_region) {
392 
393  if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
394 #ifdef DEBUG_RUNTIME_INTERNAL
395  debug(user_context) << "RegionAllocator: Freeing unused region to coalesce ("
396  << "block_ptr=" << (void *)block_region->block_ptr << " "
397  << "block_region=" << (void *)block_region << " "
398  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
399  << "block_reserved=" << (uint32_t)block->reserved << " "
400  << ")";
401 #endif
402  halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
403  MemoryRegion *memory_region = &(block_region->memory);
404  allocators.region.deallocate(user_context, memory_region);
405  block_region->memory.handle = nullptr;
406  }
407 
408  BlockRegion *prev_region = block_region->prev_ptr;
409  if (is_available(prev_region) && (prev_region != block_region)) {
410 
411 #ifdef DEBUG_RUNTIME_INTERNAL
412  debug(user_context) << "RegionAllocator: Coalescing "
413  << "previous region (offset=" << (int32_t)prev_region->memory.offset << " size=" << (int32_t)(prev_region->memory.size) << " bytes) "
414  << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)!";
415 #endif
416 
417  prev_region->next_ptr = block_region->next_ptr;
418  if (block_region->next_ptr) {
419  block_region->next_ptr->prev_ptr = prev_region;
420  }
421  prev_region->memory.size += block_region->memory.size;
422  destroy_block_region(user_context, block_region);
423  block_region = prev_region;
424  }
425 
426  BlockRegion *next_region = block_region->next_ptr;
427  if (is_available(next_region) && (next_region != block_region)) {
428 
429 #ifdef DEBUG_RUNTIME_INTERNAL
430  debug(user_context) << "RegionAllocator: Coalescing "
431  << "next region (offset=" << (int32_t)next_region->memory.offset << " size=" << (int32_t)(next_region->memory.size) << " bytes) "
432  << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)";
433 #endif
434 
435  if (next_region->next_ptr) {
436  next_region->next_ptr->prev_ptr = block_region;
437  }
438  block_region->next_ptr = next_region->next_ptr;
439  block_region->memory.size += next_region->memory.size;
440  destroy_block_region(user_context, next_region);
441  }
442 
443  return block_region;
444 }
445 
446 bool RegionAllocator::can_split(const BlockRegion *block_region, const MemoryRequest &split_request) const {
447  return (block_region && (block_region->memory.size > split_request.size) && (block_region->usage_count == 0));
448 }
449 
450 BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion *block_region, const MemoryRequest &request) {
451 
452  if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
453 #ifdef DEBUG_RUNTIME_INTERNAL
454  debug(user_context) << "RegionAllocator: Split deallocate region ("
455  << "block_ptr=" << (void *)block_region->block_ptr << " "
456  << "block_region=" << (void *)block_region << " "
457  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
458  << "block_reserved=" << (uint32_t)block_region->block_ptr->reserved << " "
459  << ")";
460 #endif
461  halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
462  MemoryRegion *memory_region = &(block_region->memory);
463  allocators.region.deallocate(user_context, memory_region);
464  block_region->memory.handle = nullptr;
465  }
466 
467  MemoryRequest split_request = request;
468  split_request.size = block_region->memory.size - request.size;
469  split_request.offset = block_region->memory.offset + request.size;
470 
471 #ifdef DEBUG_RUNTIME_INTERNAL
472  debug(user_context) << "RegionAllocator: Splitting "
473  << "current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes) "
474  << "to create empty region (offset=" << (int32_t)split_request.offset << " size=" << (int32_t)(split_request.size) << " bytes)";
475 #endif
476  BlockRegion *next_region = block_region->next_ptr;
477  BlockRegion *empty_region = create_block_region(user_context, split_request);
478  halide_abort_if_false(user_context, empty_region != nullptr);
479 
480  empty_region->next_ptr = next_region;
481  if (next_region) {
482  next_region->prev_ptr = empty_region;
483  }
484  empty_region->prev_ptr = block_region;
485  block_region->next_ptr = empty_region;
486  block_region->memory.size -= empty_region->memory.size;
487  return empty_region;
488 }
489 
490 BlockRegion *RegionAllocator::create_block_region(void *user_context, const MemoryRequest &request) {
491 #ifdef DEBUG_RUNTIME_INTERNAL
492  debug(user_context) << "RegionAllocator: Creating block region request ("
493  << "user_context=" << (void *)(user_context) << " "
494  << "offset=" << (uint32_t)request.offset << " "
495  << "size=" << (uint32_t)request.size << " "
496  << "alignment=" << (uint32_t)request.properties.alignment << " "
497  << "dedicated=" << (request.dedicated ? "true" : "false") << " "
498  << "usage=" << halide_memory_usage_name(request.properties.usage) << " "
499  << "caching=" << halide_memory_caching_name(request.properties.caching) << " "
500  << "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...";
501 #endif
502 
503  MemoryRequest region_request = request;
504  int error_code = conform(user_context, &region_request);
505  if (error_code) {
506 #ifdef DEBUG_RUNTIME_INTERNAL
507  debug(user_context) << "RegionAllocator: Failed to conform request for new block region!\n";
508 #endif
509  return nullptr;
510  }
511 
512  if (region_request.size == 0) {
513 #ifdef DEBUG_RUNTIME_INTERNAL
514  debug(user_context) << "RegionAllocator: Failed to allocate new block region ... region size was zero!\n";
515 #endif
516  return nullptr;
517  }
518 
519  BlockRegion *block_region = static_cast<BlockRegion *>(arena->reserve(user_context, true));
520  if (block_region == nullptr) {
521 #ifdef DEBUG_RUNTIME_INTERNAL
522  debug(user_context) << "RegionAllocator: Failed to allocate new block region!\n";
523 #endif
524  return nullptr;
525  }
526 
527  block_region->memory.handle = nullptr;
528  block_region->memory.offset = region_request.offset;
529  block_region->memory.size = region_request.size;
530  block_region->memory.properties = region_request.properties;
531  block_region->memory.dedicated = region_request.dedicated;
532  block_region->status = AllocationStatus::Available;
533  block_region->block_ptr = block;
534  block_region->usage_count = 0;
535 
536 #ifdef DEBUG_RUNTIME_INTERNAL
537  debug(user_context) << "RegionAllocator: Created block region allocation ("
538  << "user_context=" << (void *)(user_context) << " "
539  << "block_ptr=" << (void *)block_region->block_ptr << " "
540  << "block_region=" << (void *)block_region << " "
541  << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
542  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
543  << ")";
544 #endif
545 
546  return block_region;
547 }
548 
549 int RegionAllocator::release_block_region(void *user_context, BlockRegion *block_region) {
550 #ifdef DEBUG_RUNTIME_INTERNAL
551  debug(user_context) << "RegionAllocator: Releasing block region ("
552  << "user_context=" << (void *)(user_context) << " "
553  << "block_ptr=" << ((block_region) ? ((void *)block_region->block_ptr) : nullptr) << " "
554  << "block_region=" << (void *)block_region << " "
555  << "usage_count=" << ((block_region) ? (uint32_t)(block_region->usage_count) : 0) << " "
556  << "memory_offset=" << ((block_region) ? (uint32_t)(block_region->memory.offset) : 0) << " "
557  << "memory_size=" << ((block_region) ? (uint32_t)(block_region->memory.size) : 0) << " "
558  << "block_reserved=" << (uint32_t)(block->reserved) << ") ... ";
559 #endif
560  if (block_region == nullptr) {
561  return 0;
562  }
563 
564  if (block_region->usage_count > 0) {
565  return 0;
566  }
567 
568  if (block_region->status != AllocationStatus::Available) {
569 
570 #ifdef DEBUG_RUNTIME_INTERNAL
571  debug(user_context) << " releasing region ("
572  << "block_ptr=" << (void *)block_region->block_ptr << " "
573  << "block_region=" << (void *)block_region << " "
574  << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
575  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
576  << "block_reserved=" << (uint32_t)(block->reserved - block_region->memory.size) << " "
577  << ")";
578 #endif
579 
580  block->reserved -= block_region->memory.size;
581  }
582  block_region->status = AllocationStatus::Available;
583  return 0;
584 }
585 
586 int RegionAllocator::destroy_block_region(void *user_context, BlockRegion *block_region) {
587 #ifdef DEBUG_RUNTIME_INTERNAL
588  debug(user_context) << "RegionAllocator: Destroying block region ("
589  << "user_context=" << (void *)(user_context) << " "
590  << "block_region=" << (void *)(block_region) << ") ...";
591 #endif
592 
593  block_region->usage_count = 0;
594  release_block_region(user_context, block_region);
595  free_block_region(user_context, block_region);
596  arena->reclaim(user_context, block_region);
597  return 0;
598 }
599 
600 int RegionAllocator::alloc_block_region(void *user_context, BlockRegion *block_region) {
601 #ifdef DEBUG_RUNTIME_INTERNAL
602  debug(user_context) << "RegionAllocator: Allocating region (user_context=" << (void *)(user_context)
603  << " size=" << (int32_t)(block_region->memory.size)
604  << " offset=" << (int32_t)block_region->memory.offset << ")";
605 #endif
606  halide_abort_if_false(user_context, allocators.region.allocate != nullptr);
607  halide_abort_if_false(user_context, block_region->status == AllocationStatus::Available);
608  int error_code = 0;
609  MemoryRegion *memory_region = &(block_region->memory);
610  if (memory_region->handle == nullptr) {
611  error_code = allocators.region.allocate(user_context, memory_region);
612  memory_region->is_owner = true;
613 
614 #ifdef DEBUG_RUNTIME_INTERNAL
615  debug(user_context) << " allocating region ("
616  << "block_ptr=" << (void *)block_region->block_ptr << " "
617  << "block_region=" << (void *)block_region << " "
618  << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
619  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
620  << "block_reserved=" << (uint32_t)block->reserved << " "
621  << ")";
622 #endif
623 
624  } else {
625 
626 #ifdef DEBUG_RUNTIME_INTERNAL
627  debug(user_context) << " re-using region ("
628  << "block_ptr=" << (void *)block_region->block_ptr << " "
629  << "block_region=" << (void *)block_region << " "
630  << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
631  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
632  << "block_reserved=" << (uint32_t)block->reserved << " "
633  << ")";
634 #endif
635  }
636  block_region->status = block_region->memory.dedicated ? AllocationStatus::Dedicated : AllocationStatus::InUse;
637  block->reserved += block_region->memory.size;
638  return error_code;
639 }
640 
641 int RegionAllocator::free_block_region(void *user_context, BlockRegion *block_region) {
642 #ifdef DEBUG_RUNTIME_INTERNAL
643  debug(user_context) << "RegionAllocator: Freeing block region ("
644  << "user_context=" << (void *)(user_context) << " "
645  << "block_ptr=" << (void *)block_region->block_ptr << " "
646  << "block_region=" << (void *)(block_region) << " "
647  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
648  << "status=" << (uint32_t)block_region->status << " "
649  << "usage_count=" << (uint32_t)block_region->usage_count << " "
650  << "block_reserved=" << (uint32_t)block->reserved << ")";
651 #endif
652  if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
653 #ifdef DEBUG_RUNTIME_INTERNAL
654  debug(user_context) << " deallocating region ("
655  << "block_ptr=" << (void *)block_region->block_ptr << " "
656  << "block_region=" << (void *)block_region << " "
657  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
658  << "block_reserved=" << (uint32_t)block->reserved << " "
659  << ")";
660 #endif
661  // NOTE: Deallocate but leave memory size as is, so that coalesce can compute region merging sizes
662  halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
663  MemoryRegion *memory_region = &(block_region->memory);
664  allocators.region.deallocate(user_context, memory_region);
665  block_region->memory.handle = nullptr;
666  }
667  block_region->usage_count = 0;
668  block_region->status = AllocationStatus::Available;
669  return 0;
670 }
671 
672 int RegionAllocator::release(void *user_context) {
673 #ifdef DEBUG_RUNTIME_INTERNAL
674  debug(user_context) << "RegionAllocator: Releasing all regions ("
675  << "user_context=" << (void *)(user_context) << ") ...";
676 #endif
677 
678  BlockRegion *block_region = block->regions;
679  while (block_region != nullptr) {
680  release_block_region(user_context, block_region);
681  if (is_last_block_region(user_context, block_region)) {
682  break;
683  }
684  block_region = block_region->next_ptr;
685  }
686  return 0;
687 }
688 
689 bool RegionAllocator::collect(void *user_context) {
690 #ifdef DEBUG_RUNTIME_INTERNAL
691  debug(user_context) << "RegionAllocator: Collecting free block regions ("
692  << "user_context=" << (void *)(user_context) << ") ...";
693 
694  uint32_t collected_count = 0;
695  uint32_t remaining_count = 0;
696  uint64_t available_bytes = 0;
697  uint64_t scanned_bytes = 0;
698  uint64_t reserved = block->reserved;
699  debug(user_context) << " collecting unused regions ("
700  << "block_ptr=" << (void *)block << " "
701  << "block_reserved=" << (uint32_t)block->reserved << " "
702  << ")";
703 #endif
704 
705  bool has_collected = false;
706  BlockRegion *block_region = block->regions;
707  while (block_region != nullptr) {
708 #ifdef DEBUG_RUNTIME_INTERNAL
709  scanned_bytes += block_region->memory.size;
710  debug(user_context) << " checking region ("
711  << "block_ptr=" << (void *)block_region->block_ptr << " "
712  << "block_region=" << (void *)block_region << " "
713  << "usage_count=" << (uint32_t)(block_region->usage_count) << " "
714  << "status=" << (uint32_t)(block_region->status) << " "
715  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
716  << "block_reserved=" << (uint32_t)block->reserved << " "
717  << ")";
718 #endif
719 
720  if (can_coalesce(block_region)) {
721 #ifdef DEBUG_RUNTIME_INTERNAL
722  collected_count++;
723  debug(user_context) << " collecting region ("
724  << "block_ptr=" << (void *)block_region->block_ptr << " "
725  << "block_region=" << (void *)block_region << " "
726  << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
727  << "block_reserved=" << (uint32_t)block->reserved << " "
728  << ")";
729 #endif
730  block_region = coalesce_block_regions(user_context, block_region);
731  has_collected = true;
732  } else {
733 #ifdef DEBUG_RUNTIME_INTERNAL
734  remaining_count++;
735 #endif
736  }
737 #ifdef DEBUG_RUNTIME_INTERNAL
738  available_bytes += is_available(block_region) ? block_region->memory.size : 0;
739 #endif
740  if (is_last_block_region(user_context, block_region)) {
741  break;
742  }
743  block_region = block_region->next_ptr;
744  }
745 #ifdef DEBUG_RUNTIME_INTERNAL
746  debug(user_context) << " scanned active regions ("
747  << "block_ptr=" << (void *)block << " "
748  << "total_count=" << (uint32_t)(collected_count + remaining_count) << " "
749  << "block_reserved=" << (uint32_t)(block->reserved) << " "
750  << "scanned_bytes=" << (uint32_t)(scanned_bytes) << " "
751  << "available_bytes=" << (uint32_t)(available_bytes) << " "
752  << ")";
753 #endif
754 
755  if (has_collected) {
756 #ifdef DEBUG_RUNTIME_INTERNAL
757  debug(user_context) << " collected unused regions ("
758  << "block_ptr=" << (void *)block << " "
759  << "collected_count=" << (uint32_t)collected_count << " "
760  << "remaining_count=" << (uint32_t)remaining_count << " "
761  << "reclaimed=" << (uint32_t)(reserved - block->reserved) << " "
762  << ")";
763 #endif
764  }
765  return has_collected;
766 }
767 
768 int RegionAllocator::destroy(void *user_context) {
769 #ifdef DEBUG_RUNTIME_INTERNAL
770  debug(user_context) << "RegionAllocator: Destroying all block regions ("
771  << "user_context=" << (void *)(user_context) << ") ...";
772 #endif
773  if (block->regions != nullptr) {
774  for (BlockRegion *block_region = block->regions; block_region != nullptr;) {
775 
776  if (is_last_block_region(user_context, block_region)) {
777  destroy_block_region(user_context, block_region);
778  block_region = nullptr;
779  } else {
780  BlockRegion *prev_region = block_region;
781  block_region = block_region->next_ptr;
782  destroy_block_region(user_context, prev_region);
783  }
784  }
785  }
786  block->reserved = 0;
787  block->regions = nullptr;
788  block->allocator = nullptr;
789  if (arena != nullptr) {
790  MemoryArena::destroy(user_context, arena);
791  }
792  arena = nullptr;
793  return 0;
794 }
795 
796 bool RegionAllocator::is_compatible_block_region(const BlockRegion *block_region, const MemoryProperties &properties) const {
797  if (properties.caching != MemoryCaching::DefaultCaching) {
798  if (properties.caching != block_region->memory.properties.caching) {
799  return false;
800  }
801  }
802 
804  if (properties.visibility != block_region->memory.properties.visibility) {
805  return false;
806  }
807  }
808 
809  if (properties.usage != MemoryUsage::DefaultUsage) {
810  if (properties.usage != block_region->memory.properties.usage) {
811  return false;
812  }
813  }
814 
815  return true;
816 }
817 
818 size_t RegionAllocator::region_count(void *user_context) const {
819  if (block == nullptr) {
820  return 0;
821  }
822  size_t count = 0;
823  for (BlockRegion const *region = block->regions; !is_last_block_region(user_context, region); region = region->next_ptr) {
824  ++count;
825  }
826  return count;
827 }
828 
830  return block;
831 }
832 
833 // --
834 
835 } // namespace Internal
836 } // namespace Runtime
837 } // namespace Halide
838 
839 #endif // HALIDE_RUNTIME_REGION_ALLOCATOR_H
static int destroy(void *user_context, RegionAllocator *region_allocator)
static MemoryArena * create(void *user_context, const Config &config, const SystemMemoryAllocatorFns &allocator=default_allocator())
Definition: memory_arena.h:101
int retain(void *user_context, MemoryRegion *memory_region)
static void destroy(void *user_context, MemoryArena *arena)
Definition: memory_arena.h:115
void reclaim(void *user_context, void *ptr)
Definition: memory_arena.h:182
RegionAllocator & operator=(const RegionAllocator &)=delete
int release(void *user_context, MemoryRegion *memory_region)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline, and contains methods to using Halide&#39;s bounds tools to query properties of it.
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
#define halide_abort_if_false(user_context, cond)
int conform(void *user_context, MemoryRequest *request) const
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
unsigned __INT32_TYPE__ uint32_t
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
Not visible externally, similar to &#39;static&#39; linkage in C.
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
static constexpr uint32_t default_capacity
Definition: memory_arena.h:24
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
unsigned __INT64_TYPE__ uint64_t
WEAK const char * halide_memory_caching_name(MemoryCaching value)
int reclaim(void *user_context, MemoryRegion *memory_region)
signed __INT32_TYPE__ int32_t
void * reserve(void *user_context, bool initialize=false)
Definition: memory_arena.h:155
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)