Halide  19.0.0
Halide compiler and libraries
vulkan_memory.h
Go to the documentation of this file.
1 #ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2 #define HALIDE_RUNTIME_VULKAN_MEMORY_H
3 
6 #include "vulkan_internal.h"
7 
8 // Uncomment to enable verbose memory allocation debugging
9 // #define HL_VK_DEBUG_MEM 1
10 
11 namespace Halide {
12 namespace Runtime {
13 namespace Internal {
14 namespace Vulkan {
15 
16 // --------------------------------------------------------------------------
17 
18 // Enable external client to override Vulkan allocation callbacks (if they so desire)
20 WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
21 
22 // --------------------------------------------------------------------------
23 
24 // Runtime configuration parameters to adjust the behaviour of the block allocator
26  size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
27  size_t minimum_block_size = 32 * 1024 * 1024; //< Default block size is 32MB
28  size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
29  size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
30  size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
31 };
33 
34 // --------------------------------------------------------------------------
35 
36 /** Vulkan Memory Allocator class interface for managing large
37  * memory requests stored as contiguous blocks of memory, which
38  * are then sub-allocated into smaller regions of
39  * memory to avoid the excessive cost of vkAllocate and the limited
40  * number of available allocation calls through the API.
41  */
43 public:
44  // disable copy constructors and assignment
47 
48  // disable non-factory constrction
51 
52  // Factory methods for creation / destruction
53  static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
54  VkDevice dev, VkPhysicalDevice phys_dev,
55  const SystemMemoryAllocatorFns &system_allocator,
56  const VkAllocationCallbacks *alloc_callbacks = nullptr);
57 
58  static int destroy(void *user_context, VulkanMemoryAllocator *allocator);
59 
60  // Public interface methods
61  MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
62  int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
63  int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
64  int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
65  int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
66  bool collect(void *user_context); //< returns true if any blocks were removed
67  int release(void *user_context);
68  int destroy(void *user_context);
69 
70  void *map(void *user_context, MemoryRegion *region);
71  int unmap(void *user_context, MemoryRegion *region);
72  MemoryRegion *create_crop(void *user_context, MemoryRegion *region, uint64_t offset);
73  int destroy_crop(void *user_context, MemoryRegion *region);
74  MemoryRegion *owner_of(void *user_context, MemoryRegion *region);
75 
76  VkDevice current_device() const {
77  return this->device;
78  }
79  VkPhysicalDevice current_physical_device() const {
80  return this->physical_device;
81  }
83  return this->physical_device_limits;
84  }
86  return this->alloc_callbacks;
87  }
88 
89  static const VulkanMemoryConfig &default_config();
90 
91  static int allocate_block(void *instance_ptr, MemoryBlock *block);
92  static int deallocate_block(void *instance_ptr, MemoryBlock *block);
93  static int conform_block_request(void *instance_ptr, MemoryRequest *request);
94 
95  static int allocate_region(void *instance_ptr, MemoryRegion *region);
96  static int deallocate_region(void *instance_ptr, MemoryRegion *region);
97  static int conform_region_request(void *instance_ptr, MemoryRequest *request);
98 
99  size_t bytes_allocated_for_blocks() const;
100  size_t blocks_allocated() const;
101 
102  size_t bytes_allocated_for_regions() const;
103  size_t regions_allocated() const;
104 
105 private:
106  static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
107  static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
108 
109  // Initializes a new instance
110  int initialize(void *user_context, const VulkanMemoryConfig &config,
111  VkDevice dev, VkPhysicalDevice phys_dev,
112  const SystemMemoryAllocatorFns &system_allocator,
113  const VkAllocationCallbacks *alloc_callbacks = nullptr);
114 
115  uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
116 
117  uint32_t select_memory_type(void *user_context,
118  VkPhysicalDevice physical_device,
119  MemoryProperties properties,
120  uint32_t required_flags) const;
121 
122  int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
123 
124  size_t block_byte_count = 0;
125  size_t block_count = 0;
126  size_t region_byte_count = 0;
127  size_t region_count = 0;
128  void *owner_context = nullptr;
129  VulkanMemoryConfig config;
130  VkDevice device = nullptr;
131  VkPhysicalDevice physical_device = nullptr;
132  VkPhysicalDeviceLimits physical_device_limits = {};
133  const VkAllocationCallbacks *alloc_callbacks = nullptr;
134  BlockAllocator *block_allocator = nullptr;
135 };
136 
138  const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
139  const SystemMemoryAllocatorFns &system_allocator,
140  const VkAllocationCallbacks *alloc_callbacks) {
141 
142  if (system_allocator.allocate == nullptr) {
143  error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
144  return nullptr;
145  }
146 
147  VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
148  system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
149 
150  if (result == nullptr) {
151  error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
152  return nullptr; // caller must handle error case for out-of-memory
153  }
154 
155  result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
156  return result;
157 }
158 
159 int VulkanMemoryAllocator::destroy(void *user_context, VulkanMemoryAllocator *instance) {
160  if (instance == nullptr) {
161  error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
163  }
164  const BlockAllocator::MemoryAllocators &allocators = instance->block_allocator->current_allocators();
165  instance->destroy(user_context);
166  BlockAllocator::destroy(user_context, instance->block_allocator);
167  if (allocators.system.deallocate == nullptr) {
168  error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
170  }
171  allocators.system.deallocate(user_context, instance);
173 }
174 
175 int VulkanMemoryAllocator::initialize(void *user_context,
176  const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
177  const SystemMemoryAllocatorFns &system_allocator,
178  const VkAllocationCallbacks *callbacks) {
179 
180  owner_context = user_context;
181  config = cfg;
182  device = dev;
183  physical_device = phys_dev;
184  alloc_callbacks = callbacks;
185  region_count = 0;
186  region_byte_count = 0;
187  block_count = 0;
188  block_byte_count = 0;
190  allocators.system = system_allocator;
193  BlockAllocator::Config block_allocator_config = {0};
194  block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
195  block_allocator_config.maximum_block_count = cfg.maximum_block_count;
196  block_allocator_config.maximum_block_size = cfg.maximum_block_size;
197  block_allocator_config.minimum_block_size = cfg.minimum_block_size;
198  block_allocator_config.nearest_multiple = cfg.nearest_multiple;
199  block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
200  if (block_allocator == nullptr) {
201  error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
203  }
204 
205  // get the physical device properties to determine limits and allocation requirements
206  VkPhysicalDeviceProperties physical_device_properties = {0};
207  memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
208  vkGetPhysicalDeviceProperties(physical_device, &physical_device_properties);
209  memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
211 }
212 
213 MemoryRegion *VulkanMemoryAllocator::reserve(void *user_context, const MemoryRequest &request) {
214 #if defined(HL_VK_DEBUG_MEM)
215  debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
216  << "user_context=" << user_context << " "
217  << "block_allocator=" << (void *)(block_allocator) << " "
218  << "request_size=" << (uint32_t)(request.size) << " "
219  << "device=" << (void *)(device) << " "
220  << "physical_device=" << (void *)(physical_device) << ") ...\n";
221 #endif
222 
223  if ((device == nullptr) || (physical_device == nullptr)) {
224  error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
225  return nullptr;
226  }
227 
228  if (block_allocator == nullptr) {
229  error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
230  return nullptr;
231  }
232 
233  return block_allocator->reserve(this, request);
234 }
235 
236 void *VulkanMemoryAllocator::map(void *user_context, MemoryRegion *region) {
237 #if defined(HL_VK_DEBUG_MEM)
238  debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
239  << "user_context=" << user_context << " "
240  << "device=" << (void *)(device) << " "
241  << "physical_device=" << (void *)(physical_device) << " "
242  << "region=" << (void *)(region) << " "
243  << "region_size=" << (uint32_t)region->size << " "
244  << "region_offset=" << (uint32_t)region->offset << " "
245  << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
246 #endif
247  if ((device == nullptr) || (physical_device == nullptr)) {
248  error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
249  return nullptr;
250  }
251 
252  if (block_allocator == nullptr) {
253  error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
254  return nullptr;
255  }
256 
257  MemoryRegion *owner = owner_of(user_context, region);
258  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
259  if (region_allocator == nullptr) {
260  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
261  return nullptr; // NOTE: caller must handle nullptr
262  }
263 
264  BlockResource *block_resource = region_allocator->block_resource();
265  if (block_resource == nullptr) {
266  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
267  return nullptr; // NOTE: caller must handle nullptr
268  }
269 
270  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
271  if (device_memory == nullptr) {
272  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
273  return nullptr; // NOTE: caller must handle nullptr
274  }
275 
276  void *mapped_ptr = nullptr;
277  VkDeviceSize memory_offset = region->offset + region->range.head_offset;
278  VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
279  if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
280  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
281  return nullptr;
282  }
283 #if defined(HL_VK_DEBUG_MEM)
284  debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
285  << "user_context=" << user_context << "\n"
286  << " region_size=" << (uint32_t)region->size << "\n"
287  << " region_offset=" << (uint32_t)region->offset << "\n"
288  << " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
289  << " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
290  << " memory_offset=" << (uint32_t)memory_offset << "\n"
291  << " memory_size=" << (uint32_t)memory_size << "\n)\n";
292 #endif
293  VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
294  if (result != VK_SUCCESS) {
295  error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
296  return nullptr;
297  }
298 
299  return mapped_ptr;
300 }
301 
302 int VulkanMemoryAllocator::unmap(void *user_context, MemoryRegion *region) {
303 #if defined(HL_VK_DEBUG_MEM)
304  debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
305  << "user_context=" << user_context << " "
306  << "device=" << (void *)(device) << " "
307  << "physical_device=" << (void *)(physical_device) << " "
308  << "region=" << (void *)(region) << " "
309  << "region_size=" << (uint32_t)region->size << " "
310  << "region_offset=" << (uint32_t)region->offset << " "
311  << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
312 #endif
313  if ((device == nullptr) || (physical_device == nullptr)) {
314  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
316  }
317 
318  MemoryRegion *owner = owner_of(user_context, region);
319  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
320  if (region_allocator == nullptr) {
321  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
323  }
324 
325  BlockResource *block_resource = region_allocator->block_resource();
326  if (block_resource == nullptr) {
327  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
329  }
330 
331  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
332  if (device_memory == nullptr) {
333  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
335  }
336 
337  vkUnmapMemory(device, *device_memory);
339 }
340 
342 #if defined(HL_VK_DEBUG_MEM)
343  debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
344  << "user_context=" << user_context << " "
345  << "device=" << (void *)(device) << " "
346  << "physical_device=" << (void *)(physical_device) << " "
347  << "region=" << (void *)(region) << " "
348  << "region_size=" << (uint32_t)region->size << " "
349  << "region_offset=" << (uint32_t)region->offset << " "
350  << "crop_offset=" << (int64_t)offset << ") ...\n";
351 #endif
352  if ((device == nullptr) || (physical_device == nullptr)) {
353  error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
354  return nullptr;
355  }
356 
357  MemoryRegion *owner = owner_of(user_context, region);
358  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
359  if (region_allocator == nullptr) {
360  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
361  return nullptr; // NOTE: caller must handle nullptr
362  }
363 
364  // increment usage count
365  int error_code = region_allocator->retain(this, owner);
366  if (error_code != halide_error_code_success) {
367  error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
368  return nullptr; // NOTE: caller must handle nullptr
369  }
370 
371  // create a new region to return, and copy all the other region's properties
372  const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
373  if (allocators.system.allocate == nullptr) {
374  error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
375  return nullptr;
376  }
377 
378  MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
379  allocators.system.allocate(user_context, sizeof(MemoryRegion)));
380 
381  if (memory_region == nullptr) {
382  error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
383  return nullptr;
384  }
385  memcpy(memory_region, owner, sizeof(MemoryRegion));
386 
387  // point the handle to the owner of the allocated region, and update the head offset
388  memory_region->is_owner = false;
389  memory_region->handle = (void *)owner;
390  memory_region->range.head_offset = owner->range.head_offset + offset;
391  return memory_region;
392 }
393 
394 int VulkanMemoryAllocator::destroy_crop(void *user_context, MemoryRegion *region) {
395  if (region == nullptr) {
396  error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
398  }
399 
400  MemoryRegion *owner = owner_of(user_context, region);
401  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
402  if (region_allocator == nullptr) {
403  error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
405  }
406 
407  // decrement usage count
408  int error_code = region_allocator->release(this, owner);
409  if (error_code != halide_error_code_success) {
410  error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
411  return error_code;
412  }
413 
414  // discard the copied region struct
415  const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
416  if (allocators.system.deallocate == nullptr) {
417  error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
419  }
420  allocators.system.deallocate(user_context, region);
422 }
423 
425  if (region->is_owner) {
426  return region;
427  } else {
428  // If this is a cropped region, use the handle to retrieve the owner of the allocation
429  return reinterpret_cast<MemoryRegion *>(region->handle);
430  }
431 }
432 
433 int VulkanMemoryAllocator::release(void *user_context, MemoryRegion *region) {
434 #if defined(HL_VK_DEBUG_MEM)
435  debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
436  << "user_context=" << user_context << " "
437  << "region=" << (void *)(region) << " "
438  << "size=" << (uint32_t)region->size << " "
439  << "offset=" << (uint32_t)region->offset << ") ...\n";
440 #endif
441  if ((device == nullptr) || (physical_device == nullptr)) {
442  error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
444  }
445  if (block_allocator == nullptr) {
446  error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
448  }
449  return block_allocator->release(this, region);
450 }
451 
452 int VulkanMemoryAllocator::reclaim(void *user_context, MemoryRegion *region) {
453 #if defined(HL_VK_DEBUG_MEM)
454  debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
455  << "user_context=" << user_context << " "
456  << "region=" << (void *)(region) << " "
457  << "size=" << (uint32_t)region->size << " "
458  << "offset=" << (uint32_t)region->offset << ") ...\n";
459 #endif
460  if ((device == nullptr) || (physical_device == nullptr)) {
461  error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
463  }
464  if (block_allocator == nullptr) {
465  error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
467  }
468  return block_allocator->reclaim(this, region);
469 }
470 
471 int VulkanMemoryAllocator::retain(void *user_context, MemoryRegion *region) {
472 #if defined(HL_VK_DEBUG_MEM)
473  debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
474  << "user_context=" << user_context << " "
475  << "region=" << (void *)(region) << " "
476  << "size=" << (uint32_t)region->size << " "
477  << "offset=" << (uint32_t)region->offset << ") ...\n";
478 #endif
479  if ((device == nullptr) || (physical_device == nullptr)) {
480  error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
482  }
483  if (block_allocator == nullptr) {
484  error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
486  }
487  return block_allocator->retain(this, region);
488 }
489 
490 bool VulkanMemoryAllocator::collect(void *user_context) {
491 #if defined(HL_VK_DEBUG_MEM)
492  debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
493  << "user_context=" << user_context << ") ... \n";
494 #endif
495  if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
496  return false;
497  }
498  return block_allocator->collect(this);
499 }
500 
501 int VulkanMemoryAllocator::release(void *user_context) {
502 #if defined(HL_VK_DEBUG_MEM)
503  debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
504  << "user_context=" << user_context << ") ... \n";
505 #endif
506  if ((device == nullptr) || (physical_device == nullptr)) {
507  error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
509  }
510  if (block_allocator == nullptr) {
511  error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
513  }
514 
515  return block_allocator->release(this);
516 }
517 
518 int VulkanMemoryAllocator::destroy(void *user_context) {
519 #if defined(HL_VK_DEBUG_MEM)
520  debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
521  << "user_context=" << user_context << ") ... \n";
522 #endif
523  if (block_allocator != nullptr) {
524  block_allocator->destroy(this);
525  }
526  region_count = 0;
527  region_byte_count = 0;
528  block_count = 0;
529  block_byte_count = 0;
531 }
532 
533 const VulkanMemoryConfig &
535  static VulkanMemoryConfig result;
536  return result;
537 }
538 
539 // --
540 int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
541 #if defined(HL_VK_DEBUG_MEM)
542  debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
543  << "user_context=" << user_context << " "
544  << "size=" << (uint32_t)block->size << ", "
545  << "usage_flags=" << usage_flags << ") ... \n";
546 #endif
547  VkBufferCreateInfo create_info = {
549  nullptr, // struct extending this
550  0, // create flags
551  size, // buffer size (in bytes)
552  usage_flags, // buffer usage flags
553  VK_SHARING_MODE_EXCLUSIVE, // sharing mode
554  0, nullptr};
555 
556  // Create a buffer to determine alignment requirements
557  VkBuffer buffer = {0};
558  VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
559  if (result != VK_SUCCESS) {
560 #if defined(HL_VK_DEBUG_MEM)
561  debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
562  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
563 #endif
565  }
566 
567  vkGetBufferMemoryRequirements(this->device, buffer, memory_requirements);
568  vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
570 }
571 
572 int VulkanMemoryAllocator::conform_block_request(void *instance_ptr, MemoryRequest *request) {
573 
574  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
575  if (instance == nullptr) {
577  }
578 
579  void *user_context = instance->owner_context;
580 #if defined(HL_VK_DEBUG_MEM)
581  debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
582  << "user_context=" << user_context << " "
583  << "request=" << (void *)(request) << ") ... \n";
584 #endif
585 
586  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
587  error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
589  }
590 
591  VkMemoryRequirements memory_requirements = {0};
592  uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
593  int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
594  if (error_code != halide_error_code_success) {
595  error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
596  return error_code;
597  }
598 
599 #if defined(HL_VK_DEBUG_MEM)
600  debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
601  << "size=" << (uint32_t)request->size << ", "
602  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
603  << "required_size=" << (uint32_t)memory_requirements.size << ", "
604  << "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
605  << "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
606  << "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
607 #endif
608 
609  request->size = memory_requirements.size;
610  request->properties.alignment = memory_requirements.alignment;
612 }
613 
614 int VulkanMemoryAllocator::allocate_block(void *instance_ptr, MemoryBlock *block) {
615  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
616  if (instance == nullptr) {
618  }
619 
620  void *user_context = instance->owner_context;
621  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
622  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
624  }
625 
626  if (block == nullptr) {
627  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
629  }
630 
631 #if defined(HL_VK_DEBUG_MEM)
632  debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
633  << "user_context=" << user_context << " "
634  << "block=" << (void *)(block) << " "
635  << "size=" << (uint64_t)block->size << ", "
636  << "dedicated=" << (block->dedicated ? "true" : "false") << " "
637  << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
638  << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
639  << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
640 #endif
641 
642  // Find an appropriate memory type given the flags
643  uint32_t memory_type = instance->select_memory_type(user_context, instance->physical_device, block->properties, 0);
644  if (memory_type == invalid_memory_type) {
645  error(user_context) << "VulkanMemoryAllocator: Unable to find appropriate memory type for device!\n";
647  }
648 
649  // Allocate memory
650  VkMemoryAllocateInfo alloc_info = {
652  nullptr, // struct extending this
653  block->size, // size of allocation in bytes
654  memory_type // memory type index from physical device
655  };
656 
657  VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(nullptr, sizeof(VkDeviceMemory), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
658  if (device_memory == nullptr) {
659  debug(nullptr) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
661  }
662 
663  VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
664  if (result != VK_SUCCESS) {
665  debug(nullptr) << "VulkanMemoryAllocator: Allocation failed! vkAllocateMemory returned: " << vk_get_error_name(result) << "\n";
667  }
668 #ifdef DEBUG_RUNTIME
669  debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
670 #endif
671 
672  block->handle = (void *)device_memory;
673  instance->block_byte_count += block->size;
674  instance->block_count++;
676 }
677 
678 int VulkanMemoryAllocator::deallocate_block(void *instance_ptr, MemoryBlock *block) {
679  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
680  if (instance == nullptr) {
682  }
683 
684  void *user_context = instance->owner_context;
685 #if defined(HL_VK_DEBUG_MEM)
686  debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
687  << "user_context=" << user_context << " "
688  << "block=" << (void *)(block) << ") ... \n";
689 #endif
690 
691  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
692  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
694  }
695 
696  if (block == nullptr) {
697  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
699  }
700 
701 #if defined(HL_VK_DEBUG_MEM)
702  debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
703  << "size=" << (uint32_t)block->size << ", "
704  << "dedicated=" << (block->dedicated ? "true" : "false") << " "
705  << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
706  << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
707  << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
708 #endif
709 
710  if (block->handle == nullptr) {
711  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid handle!\n";
713  }
714 
715  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
716  if (device_memory == nullptr) {
717  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
719  }
720 
721  vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
722 #ifdef DEBUG_RUNTIME
723  debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
724 #endif
725 
726  if (instance->block_count > 0) {
727  instance->block_count--;
728  } else {
729  error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
730  instance->block_count = 0;
731  }
732 
733  if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
734  instance->block_byte_count -= block->size;
735  } else {
736  error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
737  instance->block_byte_count = 0;
738  }
739 
740  block->handle = nullptr;
741  vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
742  device_memory = nullptr;
744 }
745 
746 size_t VulkanMemoryAllocator::blocks_allocated() const {
747  return block_count;
748 }
749 
750 size_t VulkanMemoryAllocator::bytes_allocated_for_blocks() const {
751  return block_byte_count;
752 }
753 
754 uint32_t VulkanMemoryAllocator::select_memory_type(void *user_context,
755  VkPhysicalDevice physical_device,
756  MemoryProperties properties,
757  uint32_t required_flags) const {
758 
759  uint32_t want_flags = 0; //< preferred memory flags for requested access type
760  uint32_t need_flags = 0; //< must have in order to enable requested access
761  switch (properties.visibility) {
764  break;
767  break;
771  break;
774  break;
777  default:
778  error(nullptr) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
779  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
780  return invalid_memory_type;
781  };
782 
783  switch (properties.caching) {
785  if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
787  }
788  break;
790  if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
792  }
793  break;
795  if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
797  }
798  break;
801  break;
803  default:
804  error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
805  << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
806  return invalid_memory_type;
807  };
808 
809  VkPhysicalDeviceMemoryProperties device_memory_properties;
810  vkGetPhysicalDeviceMemoryProperties(physical_device, &device_memory_properties);
811 
812  uint32_t result = invalid_memory_type;
813  for (uint32_t i = 0; i < device_memory_properties.memoryTypeCount; ++i) {
814 
815  // if required flags are given, see if the memory type matches the requirement
816  if (required_flags) {
817  if (((required_flags >> i) & 1) == 0) {
818  continue;
819  }
820  }
821 
822  const VkMemoryPropertyFlags properties = device_memory_properties.memoryTypes[i].propertyFlags;
823  if (need_flags) {
824  if ((properties & need_flags) != need_flags) {
825  continue;
826  }
827  }
828 
829  if (want_flags) {
830  if ((properties & want_flags) != want_flags) {
831  continue;
832  }
833  }
834 
835  result = i;
836  break;
837  }
838 
839  if (result == invalid_memory_type) {
840  error(user_context) << "VulkanBlockAllocator: Failed to find appropriate memory type for given properties:\n\t"
841  << "usage=" << halide_memory_usage_name(properties.usage) << " "
842  << "caching=" << halide_memory_caching_name(properties.caching) << " "
843  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
844  return invalid_memory_type;
845  }
846 
847  return result;
848 }
849 
850 // --
851 
852 int VulkanMemoryAllocator::conform(void *user_context, MemoryRequest *request) {
853 
854  // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
855  // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
856  // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
857  // whose size may be required to larger than our requested size (even though we will only ever touch the
858  // size of the region we're managing as within our block)
859 
860  VkMemoryRequirements memory_requirements = {0};
861  uint32_t usage_flags = select_memory_usage(user_context, request->properties);
862  int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
863  if (error_code != halide_error_code_success) {
864  error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
865  return error_code;
866  }
867 
868 #if defined(HL_VK_DEBUG_MEM)
869  debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
870  << "requested_size=" << (uint32_t)region->size << ", "
871  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
872  << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
873 #endif
874 
875  // Enforce any alignment constraints reported by the device limits for each usage type
876  if (usage_flags & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) {
877  if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
878  request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
879  }
880  } else if (usage_flags & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) {
881  if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
882  request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
883  }
884  }
885 
886  // Ensure the request ends on an aligned address
887  if (request->alignment > config.nearest_multiple) {
888  request->properties.nearest_multiple = request->alignment;
889  }
890 
891  size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
892  size_t actual_offset = aligned_offset(request->offset, actual_alignment);
893  size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
894 
895 #if defined(HL_VK_DEBUG_MEM)
896  if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
897  debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
898  << " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
899  << " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
900  << " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
901  << " required.size = " << (uint64_t)memory_requirements.size << ",\n"
902  << " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
903  }
904 #endif
905  request->size = actual_size;
906  request->alignment = actual_alignment;
907  request->offset = actual_offset;
908 
910 }
911 
912 int VulkanMemoryAllocator::conform_region_request(void *instance_ptr, MemoryRequest *request) {
913 
914  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
915  if (instance == nullptr) {
917  }
918 
919  void *user_context = instance->owner_context;
920 #if defined(HL_VK_DEBUG_MEM)
921  debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
922  << "user_context=" << user_context << " "
923  << "request=" << (void *)(region) << ") ... \n";
924 #endif
925 
926  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
927  error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
929  }
930 
931 #if defined(HL_VK_DEBUG_MEM)
932  debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
933  << "size=" << (uint32_t)request->size << ", "
934  << "offset=" << (uint32_t)request->offset << ", "
935  << "dedicated=" << (request->dedicated ? "true" : "false") << " "
936  << "usage=" << halide_memory_usage_name(request->properties.usage) << " "
937  << "caching=" << halide_memory_caching_name(request->properties.caching) << " "
938  << "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
939 #endif
940 
941  return instance->conform(user_context, request);
942 }
943 
944 int VulkanMemoryAllocator::allocate_region(void *instance_ptr, MemoryRegion *region) {
945 
946  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
947  if (instance == nullptr) {
949  }
950 
951  void *user_context = instance->owner_context;
952 #if defined(HL_VK_DEBUG_MEM)
953  debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
954  << "user_context=" << user_context << " "
955  << "region=" << (void *)(region) << ") ... \n";
956 #endif
957 
958  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
959  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
961  }
962 
963  if (region == nullptr) {
964  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
966  }
967 
968 #if defined(HL_VK_DEBUG_MEM)
969  debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
970  << "size=" << (uint32_t)region->size << ", "
971  << "offset=" << (uint32_t)region->offset << ", "
972  << "dedicated=" << (region->dedicated ? "true" : "false") << " "
973  << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
974  << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
975  << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
976 #endif
977 
978  uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
979 
980  VkBufferCreateInfo create_info = {
982  nullptr, // struct extending this
983  0, // create flags
984  region->size, // buffer size (in bytes)
985  usage_flags, // buffer usage flags
986  VK_SHARING_MODE_EXCLUSIVE, // sharing mode
987  0, nullptr};
988 
989  VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
990  if (buffer == nullptr) {
991  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
993  }
994 
995  VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
996  if (result != VK_SUCCESS) {
997  error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
998  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1000  }
1001 
1002  // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
1003  // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
1004  // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
1005  // whose size may be required to larger than our requested size (even though we will only ever touch the
1006  // size of the region we're managing as within our block)
1007  VkMemoryRequirements memory_requirements = {0};
1008  vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
1009 
1010 #if defined(HL_VK_DEBUG_MEM)
1011  debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1012  << "requested_size=" << (uint32_t)region->size << ", "
1013  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1014  << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1015 #endif
1016 
1017  if (memory_requirements.size > region->size) {
1018  vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1019 #ifdef DEBUG_RUNTIME
1020  debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
1021  << (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
1022 #endif
1023  create_info.size = memory_requirements.size;
1024  VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1025  if (result != VK_SUCCESS) {
1026  error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1027  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1029  }
1030  }
1031 
1032 #ifdef DEBUG_RUNTIME
1033  debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1034 #endif
1035 
1036  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, region);
1037  if (region_allocator == nullptr) {
1038  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1040  }
1041 
1042  BlockResource *block_resource = region_allocator->block_resource();
1043  if (block_resource == nullptr) {
1044  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1046  }
1047 
1048  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
1049  if (device_memory == nullptr) {
1050  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1052  }
1053 
1054  // Finally, bind buffer to the device memory
1055  result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
1056  if (result != VK_SUCCESS) {
1057  error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
1058  << "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
1060  }
1061 
1062  region->handle = (void *)buffer;
1063  region->is_owner = true;
1064  instance->region_byte_count += region->size;
1065  instance->region_count++;
1067 }
1068 
1069 int VulkanMemoryAllocator::deallocate_region(void *instance_ptr, MemoryRegion *region) {
1070  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1071  if (instance == nullptr) {
1073  }
1074 
1075  void *user_context = instance->owner_context;
1076 #if defined(HL_VK_DEBUG_MEM)
1077  debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
1078  << "user_context=" << user_context << " "
1079  << "region=" << (void *)(region) << ") ... \n";
1080 #endif
1081 
1082  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1083  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1085  }
1086 
1087  if (region == nullptr) {
1088  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1090  }
1091 
1092 #if defined(HL_VK_DEBUG_MEM)
1093  debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
1094  << "size=" << (uint32_t)region->size << ", "
1095  << "offset=" << (uint32_t)region->offset << ", "
1096  << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1097  << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1098  << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1099  << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1100 #endif
1101 
1102  if (region->handle == nullptr) {
1103  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1105  }
1106 
1107  VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
1108  if (buffer == nullptr) {
1109  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1111  }
1112 
1113  vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1114 #ifdef DEBUG_RUNTIME
1115  debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1116 #endif
1117  region->handle = nullptr;
1118  if (instance->region_count > 0) {
1119  instance->region_count--;
1120  } else {
1121  error(nullptr) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1122  instance->region_count = 0;
1124  }
1125 
1126  if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
1127  instance->region_byte_count -= region->size;
1128  } else {
1129  error(nullptr) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1130  instance->region_byte_count = 0;
1132  }
1133  vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1134  buffer = nullptr;
1136 }
1137 
1138 size_t VulkanMemoryAllocator::regions_allocated() const {
1139  return region_count;
1140 }
1141 
1142 size_t VulkanMemoryAllocator::bytes_allocated_for_regions() const {
1143  return region_byte_count;
1144 }
1145 
1146 uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
1147  uint32_t result = 0;
1148  switch (properties.usage) {
1151  break;
1155  break;
1158  break;
1161  break;
1164  break;
1167  default:
1168  error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1169  << "usage=" << halide_memory_usage_name(properties.usage) << "\n";
1170  return invalid_usage_flags;
1171  };
1172 
1173  if (result == invalid_usage_flags) {
1174  error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1175  << "usage=" << halide_memory_usage_name(properties.usage) << " "
1176  << "caching=" << halide_memory_caching_name(properties.caching) << " "
1177  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
1178  return invalid_usage_flags;
1179  }
1180 
1181  return result;
1182 }
1183 
1184 // --------------------------------------------------------------------------
1185 
1186 namespace {
1187 
1188 // --------------------------------------------------------------------------
1189 // Halide System allocator for host allocations
1190 void *vk_system_malloc(void *user_context, size_t size) {
1191  return malloc(size);
1192 }
1193 
1194 void vk_system_free(void *user_context, void *ptr) {
1195  free(ptr);
1196 }
1197 
1198 // Vulkan host-side allocation
1199 void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
1200  if (callbacks) {
1201  return callbacks->pfnAllocation(user_context, size, alignment, scope);
1202  } else {
1203  return vk_system_malloc(user_context, size);
1204  }
1205 }
1206 
1207 void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
1208  if (callbacks) {
1209  return callbacks->pfnFree(user_context, ptr);
1210  } else {
1211  return vk_system_free(user_context, ptr);
1212  }
1213 }
1214 
1215 VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
1216  VkDevice device,
1217  VkPhysicalDevice physical_device,
1218  const VkAllocationCallbacks *alloc_callbacks) {
1219 
1220  SystemMemoryAllocatorFns system_allocator = {vk_system_malloc, vk_system_free};
1221  VulkanMemoryConfig config = memory_allocator_config;
1222 
1223  // Parse the allocation config string (if specified).
1224  //
1225  // `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
1226  // allocator use the given constraints specified as three integer values
1227  // separated by a `:` or `;`. These values correspond to `minimum_block_size`,
1228  // `maximum_block_size` and `maximum_block_count`.
1229  //
1230  const char *alloc_config = vk_get_alloc_config_internal(user_context);
1232  StringTable alloc_config_values;
1233  alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
1234  if (alloc_config_values.size() > 0) {
1235  config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
1236  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
1237  }
1238  if (alloc_config_values.size() > 1) {
1239  config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
1240  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
1241  }
1242  if (alloc_config_values.size() > 2) {
1243  config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
1244  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
1245  }
1246  if (alloc_config_values.size() > 3) {
1247  config.maximum_block_count = atoi(alloc_config_values[3]);
1248  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
1249  }
1250  if (alloc_config_values.size() > 4) {
1251  config.nearest_multiple = atoi(alloc_config_values[4]);
1252  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
1253  }
1254  }
1255 
1256  return VulkanMemoryAllocator::create(user_context,
1257  config, device, physical_device,
1258  system_allocator, alloc_callbacks);
1259 }
1260 
1261 int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
1262  if (allocator != nullptr) {
1263  VulkanMemoryAllocator::destroy(user_context, allocator);
1264  allocator = nullptr;
1265  }
1267 }
1268 
1269 // --------------------------------------------------------------------------
1270 
1271 int vk_clear_device_buffer(void *user_context,
1272  VulkanMemoryAllocator *allocator,
1273  VkCommandPool command_pool,
1274  VkQueue command_queue,
1275  VkBuffer device_buffer) {
1276 
1277 #ifdef DEBUG_RUNTIME
1278  debug(user_context)
1279  << " vk_clear_device_buffer (user_context: " << user_context << ", "
1280  << "allocator: " << (void *)allocator << ", "
1281  << "command_pool: " << (void *)command_pool << ", "
1282  << "command_queue: " << (void *)command_queue << ", "
1283  << "device_buffer: " << (void *)device_buffer << ")\n";
1284 #endif
1285 
1286  // create a command buffer
1287  VkCommandBuffer command_buffer;
1288  int error_code = vk_create_command_buffer(user_context, allocator, command_pool, &command_buffer);
1289  if (error_code != halide_error_code_success) {
1290  error(user_context) << "Vulkan: Failed to create command buffer!\n";
1291  return error_code;
1292  }
1293 
1294  // begin the command buffer
1295  VkCommandBufferBeginInfo command_buffer_begin_info =
1296  {
1298  nullptr, // pointer to struct extending this
1300  nullptr // pointer to parent command buffer
1301  };
1302 
1303  VkResult result = vkBeginCommandBuffer(command_buffer, &command_buffer_begin_info);
1304  if (result != VK_SUCCESS) {
1305  error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
1307  }
1308 
1309  // fill buffer with zero values up to the size of the buffer
1310  vkCmdFillBuffer(command_buffer, device_buffer, 0, VK_WHOLE_SIZE, 0);
1311 
1312  // end the command buffer
1313  result = vkEndCommandBuffer(command_buffer);
1314  if (result != VK_SUCCESS) {
1315  error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
1317  }
1318 
1319  // submit the command buffer
1320  VkSubmitInfo submit_info =
1321  {
1322  VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
1323  nullptr, // pointer to struct extending this
1324  0, // wait semaphore count
1325  nullptr, // semaphores
1326  nullptr, // pipeline stages where semaphore waits occur
1327  1, // how many command buffers to execute
1328  &command_buffer, // the command buffers
1329  0, // number of semaphores to signal
1330  nullptr // the semaphores to signal
1331  };
1332 
1333  result = vkQueueSubmit(command_queue, 1, &submit_info, 0);
1334  if (result != VK_SUCCESS) {
1335  error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
1337  }
1338 
1339  // wait for memset to finish
1340  result = vkQueueWaitIdle(command_queue);
1341  if (result != VK_SUCCESS) {
1342  error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
1344  }
1345 
1346  error_code = vk_destroy_command_buffer(user_context, allocator, command_pool, command_buffer);
1347  if (error_code != halide_error_code_success) {
1348  error(user_context) << "Vulkan: Failed to destroy command buffer!\n";
1349  return error_code;
1350  }
1351 
1353 }
1354 
1355 // --------------------------------------------------------------------------
1356 
1357 } // namespace
1358 } // namespace Vulkan
1359 } // namespace Internal
1360 } // namespace Runtime
1361 } // namespace Halide
1362 
1363 // --------------------------------------------------------------------------
1364 
1365 extern "C" {
1366 
1367 // --------------------------------------------------------------------------
1368 
1370  using namespace Halide::Runtime::Internal::Vulkan;
1372  custom_allocation_callbacks = callbacks;
1373 }
1374 
1376  using namespace Halide::Runtime::Internal::Vulkan;
1379 }
1380 
1381 // --------------------------------------------------------------------------
1382 
1383 } // extern "C"
1384 
1385 #endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
@ halide_error_code_generic_error
An uncategorized error occurred.
@ halide_error_code_success
There was no error.
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static void destroy(void *user_context, BlockAllocator *block_allocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
int reclaim(void *user_context, MemoryRegion *region)
const MemoryAllocators & current_allocators() const
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
int retain(void *user_context, MemoryRegion *memory_region)
int release(void *user_context, MemoryRegion *memory_region)
size_t parse(void *user_context, const char *str, const char *delim)
Definition: string_table.h:159
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
Definition: vulkan_memory.h:42
int reclaim(void *user_context, MemoryRegion *region)
static const VulkanMemoryConfig & default_config()
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
int release(void *user_context, MemoryRegion *region)
static int allocate_block(void *instance_ptr, MemoryBlock *block)
int conform(void *user_context, MemoryRequest *request)
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
int unmap(void *user_context, MemoryRegion *region)
const VkAllocationCallbacks * callbacks() const
Definition: vulkan_memory.h:85
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
int destroy_crop(void *user_context, MemoryRegion *region)
static int allocate_region(void *instance_ptr, MemoryRegion *region)
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
VkPhysicalDeviceLimits current_physical_device_limits() const
Definition: vulkan_memory.h:82
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
void * map(void *user_context, MemoryRegion *region)
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
int retain(void *user_context, MemoryRegion *region)
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize memoryOffset)
VkFlags VkMemoryPropertyFlags
Definition: mini_vulkan.h:1126
VKAPI_ATTR void VKAPI_CALL vkCmdFillBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDeviceSize dstOffset, VkDeviceSize size, uint32_t data)
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMemory)
#define VK_MAX_MEMORY_TYPES
Definition: mini_vulkan.h:125
@ VK_SHARING_MODE_EXCLUSIVE
Definition: mini_vulkan.h:627
@ VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
Definition: mini_vulkan.h:1121
@ VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT
Definition: mini_vulkan.h:1119
@ VK_MEMORY_PROPERTY_HOST_CACHED_BIT
Definition: mini_vulkan.h:1122
@ VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
Definition: mini_vulkan.h:1120
VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice device, VkBuffer buffer, VkMemoryRequirements *pMemoryRequirements)
VkSystemAllocationScope
Definition: mini_vulkan.h:365
@ VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
Definition: mini_vulkan.h:367
VKAPI_ATTR VkResult VKAPI_CALL vkQueueWaitIdle(VkQueue queue)
uint64_t VkDeviceSize
Definition: mini_vulkan.h:71
VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks *pAllocator)
@ VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT
Definition: mini_vulkan.h:1372
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceProperties *pProperties)
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceMemoryProperties *pMemoryProperties)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer)
VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence)
VKAPI_ATTR VkResult VKAPI_CALL vkMapMemory(VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags, void **ppData)
#define VK_WHOLE_SIZE
Definition: mini_vulkan.h:117
@ VK_BUFFER_USAGE_TRANSFER_DST_BIT
Definition: mini_vulkan.h:1228
@ VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT
Definition: mini_vulkan.h:1231
@ VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
Definition: mini_vulkan.h:1232
@ VK_BUFFER_USAGE_TRANSFER_SRC_BIT
Definition: mini_vulkan.h:1227
VkResult
Definition: mini_vulkan.h:138
@ VK_SUCCESS
Definition: mini_vulkan.h:139
VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice device, VkDeviceMemory memory)
VKAPI_ATTR VkResult VKAPI_CALL vkEndCommandBuffer(VkCommandBuffer commandBuffer)
VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks *pAllocator)
@ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO
Definition: mini_vulkan.h:185
@ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO
Definition: mini_vulkan.h:215
@ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO
Definition: mini_vulkan.h:178
@ VK_STRUCTURE_TYPE_SUBMIT_INFO
Definition: mini_vulkan.h:177
VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo)
void destroy(const T *t)
WEAK VulkanMemoryConfig memory_allocator_config
Definition: vulkan_memory.h:32
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
Definition: vulkan_memory.h:20
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
Definition: vulkan_memory.h:19
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
void * malloc(size_t)
int atoi(const char *)
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
#define WEAK
void * memcpy(void *s1, const void *s2, size_t n)
void free(void *)
static bool is_empty(const char *str)
PFN_vkAllocationFunction pfnAllocation
Definition: mini_vulkan.h:1451
PFN_vkFreeFunction pfnFree
Definition: mini_vulkan.h:1453
VkDeviceSize size
Definition: mini_vulkan.h:1864
VkDeviceSize alignment
Definition: mini_vulkan.h:1753
VkMemoryPropertyFlags propertyFlags
Definition: mini_vulkan.h:1673
VkDeviceSize minStorageBufferOffsetAlignment
Definition: mini_vulkan.h:1606
VkDeviceSize minUniformBufferOffsetAlignment
Definition: mini_vulkan.h:1605
VkMemoryType memoryTypes[VK_MAX_MEMORY_TYPES]
Definition: mini_vulkan.h:1684
VkPhysicalDeviceLimits limits
Definition: mini_vulkan.h:1661
#define HL_VK_ENV_DELIM
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)