Halide  17.0.2
Halide compiler and libraries
vulkan_memory.h
Go to the documentation of this file.
1 #ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2 #define HALIDE_RUNTIME_VULKAN_MEMORY_H
3 
6 #include "vulkan_internal.h"
7 
8 // Uncomment to enable verbose memory allocation debugging
9 // #define HL_VK_DEBUG_MEM 1
10 
11 namespace Halide {
12 namespace Runtime {
13 namespace Internal {
14 namespace Vulkan {
15 
16 // --------------------------------------------------------------------------
17 
18 // Enable external client to override Vulkan allocation callbacks (if they so desire)
20 WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
21 
22 // --------------------------------------------------------------------------
23 
24 // Runtime configuration parameters to adjust the behaviour of the block allocator
26  size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
27  size_t minimum_block_size = 32 * 1024 * 1024; //< Default block size is 32MB
28  size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
29  size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
30  size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
31 };
33 
34 // --------------------------------------------------------------------------
35 
36 /** Vulkan Memory Allocator class interface for managing large
37  * memory requests stored as contiguous blocks of memory, which
38  * are then sub-allocated into smaller regions of
39  * memory to avoid the excessive cost of vkAllocate and the limited
40  * number of available allocation calls through the API.
41  */
43 public:
44  // disable copy constructors and assignment
47 
48  // disable non-factory constrction
49  VulkanMemoryAllocator() = delete;
50  ~VulkanMemoryAllocator() = delete;
51 
52  // Factory methods for creation / destruction
53  static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
54  VkDevice dev, VkPhysicalDevice phys_dev,
55  const SystemMemoryAllocatorFns &system_allocator,
56  const VkAllocationCallbacks *alloc_callbacks = nullptr);
57 
58  static int destroy(void *user_context, VulkanMemoryAllocator *allocator);
59 
60  // Public interface methods
61  MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
62  int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
63  int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
64  int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
65  int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
66  bool collect(void *user_context); //< returns true if any blocks were removed
67  int release(void *user_context);
68  int destroy(void *user_context);
69 
70  void *map(void *user_context, MemoryRegion *region);
71  int unmap(void *user_context, MemoryRegion *region);
72  MemoryRegion *create_crop(void *user_context, MemoryRegion *region, uint64_t offset);
73  int destroy_crop(void *user_context, MemoryRegion *region);
74  MemoryRegion *owner_of(void *user_context, MemoryRegion *region);
75 
76  VkDevice current_device() const {
77  return this->device;
78  }
79  VkPhysicalDevice current_physical_device() const {
80  return this->physical_device;
81  }
83  return this->alloc_callbacks;
84  }
85 
86  static const VulkanMemoryConfig &default_config();
87 
88  static int allocate_block(void *instance_ptr, MemoryBlock *block);
89  static int deallocate_block(void *instance_ptr, MemoryBlock *block);
90  static int conform_block_request(void *instance_ptr, MemoryRequest *request);
91 
92  static int allocate_region(void *instance_ptr, MemoryRegion *region);
93  static int deallocate_region(void *instance_ptr, MemoryRegion *region);
94  static int conform_region_request(void *instance_ptr, MemoryRequest *request);
95 
96  size_t bytes_allocated_for_blocks() const;
97  size_t blocks_allocated() const;
98 
99  size_t bytes_allocated_for_regions() const;
100  size_t regions_allocated() const;
101 
102 private:
103  static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
104  static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
105 
106  // Initializes a new instance
107  int initialize(void *user_context, const VulkanMemoryConfig &config,
108  VkDevice dev, VkPhysicalDevice phys_dev,
109  const SystemMemoryAllocatorFns &system_allocator,
110  const VkAllocationCallbacks *alloc_callbacks = nullptr);
111 
112  uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
113 
114  uint32_t select_memory_type(void *user_context,
115  VkPhysicalDevice physical_device,
116  MemoryProperties properties,
117  uint32_t required_flags) const;
118 
119  int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
120 
121  size_t block_byte_count = 0;
122  size_t block_count = 0;
123  size_t region_byte_count = 0;
124  size_t region_count = 0;
125  void *owner_context = nullptr;
126  VulkanMemoryConfig config;
127  VkDevice device = nullptr;
128  VkPhysicalDevice physical_device = nullptr;
129  VkPhysicalDeviceLimits physical_device_limits = {};
130  const VkAllocationCallbacks *alloc_callbacks = nullptr;
131  BlockAllocator *block_allocator = nullptr;
132 };
133 
135  const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
136  const SystemMemoryAllocatorFns &system_allocator,
137  const VkAllocationCallbacks *alloc_callbacks) {
138 
139  if (system_allocator.allocate == nullptr) {
140  error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
141  return nullptr;
142  }
143 
144  VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
145  system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
146 
147  if (result == nullptr) {
148  error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
149  return nullptr; // caller must handle error case for out-of-memory
150  }
151 
152  result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
153  return result;
154 }
155 
156 int VulkanMemoryAllocator::destroy(void *user_context, VulkanMemoryAllocator *instance) {
157  if (instance == nullptr) {
158  error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
160  }
161  const BlockAllocator::MemoryAllocators &allocators = instance->block_allocator->current_allocators();
162  instance->destroy(user_context);
163  BlockAllocator::destroy(user_context, instance->block_allocator);
164  if (allocators.system.deallocate == nullptr) {
165  error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
167  }
168  allocators.system.deallocate(user_context, instance);
170 }
171 
172 int VulkanMemoryAllocator::initialize(void *user_context,
173  const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
174  const SystemMemoryAllocatorFns &system_allocator,
175  const VkAllocationCallbacks *callbacks) {
176 
177  owner_context = user_context;
178  config = cfg;
179  device = dev;
180  physical_device = phys_dev;
181  alloc_callbacks = callbacks;
182  region_count = 0;
183  region_byte_count = 0;
184  block_count = 0;
185  block_byte_count = 0;
187  allocators.system = system_allocator;
190  BlockAllocator::Config block_allocator_config = {0};
191  block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
192  block_allocator_config.maximum_block_count = cfg.maximum_block_count;
193  block_allocator_config.maximum_block_size = cfg.maximum_block_size;
194  block_allocator_config.minimum_block_size = cfg.minimum_block_size;
195  block_allocator_config.nearest_multiple = cfg.nearest_multiple;
196  block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
197  if (block_allocator == nullptr) {
198  error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
200  }
201 
202  // get the physical device properties to determine limits and allocation requirements
203  VkPhysicalDeviceProperties physical_device_properties = {0};
204  memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
205  vkGetPhysicalDeviceProperties(physical_device, &physical_device_properties);
206  memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
208 }
209 
210 MemoryRegion *VulkanMemoryAllocator::reserve(void *user_context, const MemoryRequest &request) {
211 #if defined(HL_VK_DEBUG_MEM)
212  debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
213  << "user_context=" << user_context << " "
214  << "block_allocator=" << (void *)(block_allocator) << " "
215  << "request_size=" << (uint32_t)(request.size) << " "
216  << "device=" << (void *)(device) << " "
217  << "physical_device=" << (void *)(physical_device) << ") ...\n";
218 #endif
219 
220  if ((device == nullptr) || (physical_device == nullptr)) {
221  error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
222  return nullptr;
223  }
224 
225  if (block_allocator == nullptr) {
226  error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
227  return nullptr;
228  }
229 
230  return block_allocator->reserve(this, request);
231 }
232 
233 void *VulkanMemoryAllocator::map(void *user_context, MemoryRegion *region) {
234 #if defined(HL_VK_DEBUG_MEM)
235  debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
236  << "user_context=" << user_context << " "
237  << "device=" << (void *)(device) << " "
238  << "physical_device=" << (void *)(physical_device) << " "
239  << "region=" << (void *)(region) << " "
240  << "region_size=" << (uint32_t)region->size << " "
241  << "region_offset=" << (uint32_t)region->offset << " "
242  << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
243 #endif
244  if ((device == nullptr) || (physical_device == nullptr)) {
245  error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
246  return nullptr;
247  }
248 
249  if (block_allocator == nullptr) {
250  error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
251  return nullptr;
252  }
253 
254  MemoryRegion *owner = owner_of(user_context, region);
255  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
256  if (region_allocator == nullptr) {
257  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
258  return nullptr; // NOTE: caller must handle nullptr
259  }
260 
261  BlockResource *block_resource = region_allocator->block_resource();
262  if (block_resource == nullptr) {
263  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
264  return nullptr; // NOTE: caller must handle nullptr
265  }
266 
267  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
268  if (device_memory == nullptr) {
269  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
270  return nullptr; // NOTE: caller must handle nullptr
271  }
272 
273  void *mapped_ptr = nullptr;
274  VkDeviceSize memory_offset = region->offset + region->range.head_offset;
275  VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
276  if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
277  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
278  return nullptr;
279  }
280 #if defined(HL_VK_DEBUG_MEM)
281  debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
282  << "user_context=" << user_context << "\n"
283  << " region_size=" << (uint32_t)region->size << "\n"
284  << " region_offset=" << (uint32_t)region->offset << "\n"
285  << " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
286  << " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
287  << " memory_offset=" << (uint32_t)memory_offset << "\n"
288  << " memory_size=" << (uint32_t)memory_size << "\n)\n";
289 #endif
290  VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
291  if (result != VK_SUCCESS) {
292  error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
293  return nullptr;
294  }
295 
296  return mapped_ptr;
297 }
298 
299 int VulkanMemoryAllocator::unmap(void *user_context, MemoryRegion *region) {
300 #if defined(HL_VK_DEBUG_MEM)
301  debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
302  << "user_context=" << user_context << " "
303  << "device=" << (void *)(device) << " "
304  << "physical_device=" << (void *)(physical_device) << " "
305  << "region=" << (void *)(region) << " "
306  << "region_size=" << (uint32_t)region->size << " "
307  << "region_offset=" << (uint32_t)region->offset << " "
308  << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
309 #endif
310  if ((device == nullptr) || (physical_device == nullptr)) {
311  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
313  }
314 
315  MemoryRegion *owner = owner_of(user_context, region);
316  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
317  if (region_allocator == nullptr) {
318  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
320  }
321 
322  BlockResource *block_resource = region_allocator->block_resource();
323  if (block_resource == nullptr) {
324  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
326  }
327 
328  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
329  if (device_memory == nullptr) {
330  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
332  }
333 
334  vkUnmapMemory(device, *device_memory);
336 }
337 
339 #if defined(HL_VK_DEBUG_MEM)
340  debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
341  << "user_context=" << user_context << " "
342  << "device=" << (void *)(device) << " "
343  << "physical_device=" << (void *)(physical_device) << " "
344  << "region=" << (void *)(region) << " "
345  << "region_size=" << (uint32_t)region->size << " "
346  << "region_offset=" << (uint32_t)region->offset << " "
347  << "crop_offset=" << (int64_t)offset << ") ...\n";
348 #endif
349  if ((device == nullptr) || (physical_device == nullptr)) {
350  error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
351  return nullptr;
352  }
353 
354  MemoryRegion *owner = owner_of(user_context, region);
355  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
356  if (region_allocator == nullptr) {
357  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
358  return nullptr; // NOTE: caller must handle nullptr
359  }
360 
361  // increment usage count
362  int error_code = region_allocator->retain(this, owner);
363  if (error_code != halide_error_code_success) {
364  error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
365  return nullptr; // NOTE: caller must handle nullptr
366  }
367 
368  // create a new region to return, and copy all the other region's properties
369  const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
370  if (allocators.system.allocate == nullptr) {
371  error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
372  return nullptr;
373  }
374 
375  MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
376  allocators.system.allocate(user_context, sizeof(MemoryRegion)));
377 
378  if (memory_region == nullptr) {
379  error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
380  return nullptr;
381  }
382  memcpy(memory_region, owner, sizeof(MemoryRegion));
383 
384  // point the handle to the owner of the allocated region, and update the head offset
385  memory_region->is_owner = false;
386  memory_region->handle = (void *)owner;
387  memory_region->range.head_offset = owner->range.head_offset + offset;
388  return memory_region;
389 }
390 
391 int VulkanMemoryAllocator::destroy_crop(void *user_context, MemoryRegion *region) {
392  if (region == nullptr) {
393  error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
395  }
396 
397  MemoryRegion *owner = owner_of(user_context, region);
398  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
399  if (region_allocator == nullptr) {
400  error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
402  }
403 
404  // decrement usage count
405  int error_code = region_allocator->release(this, owner);
406  if (error_code != halide_error_code_success) {
407  error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
408  return error_code;
409  }
410 
411  // discard the copied region struct
412  const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
413  if (allocators.system.deallocate == nullptr) {
414  error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
416  }
417  allocators.system.deallocate(user_context, region);
419 }
420 
422  if (region->is_owner) {
423  return region;
424  } else {
425  // If this is a cropped region, use the handle to retrieve the owner of the allocation
426  return reinterpret_cast<MemoryRegion *>(region->handle);
427  }
428 }
429 
430 int VulkanMemoryAllocator::release(void *user_context, MemoryRegion *region) {
431 #if defined(HL_VK_DEBUG_MEM)
432  debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
433  << "user_context=" << user_context << " "
434  << "region=" << (void *)(region) << " "
435  << "size=" << (uint32_t)region->size << " "
436  << "offset=" << (uint32_t)region->offset << ") ...\n";
437 #endif
438  if ((device == nullptr) || (physical_device == nullptr)) {
439  error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
441  }
442  if (block_allocator == nullptr) {
443  error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
445  }
446  return block_allocator->release(this, region);
447 }
448 
449 int VulkanMemoryAllocator::reclaim(void *user_context, MemoryRegion *region) {
450 #if defined(HL_VK_DEBUG_MEM)
451  debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
452  << "user_context=" << user_context << " "
453  << "region=" << (void *)(region) << " "
454  << "size=" << (uint32_t)region->size << " "
455  << "offset=" << (uint32_t)region->offset << ") ...\n";
456 #endif
457  if ((device == nullptr) || (physical_device == nullptr)) {
458  error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
460  }
461  if (block_allocator == nullptr) {
462  error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
464  }
465  return block_allocator->reclaim(this, region);
466 }
467 
468 int VulkanMemoryAllocator::retain(void *user_context, MemoryRegion *region) {
469 #if defined(HL_VK_DEBUG_MEM)
470  debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
471  << "user_context=" << user_context << " "
472  << "region=" << (void *)(region) << " "
473  << "size=" << (uint32_t)region->size << " "
474  << "offset=" << (uint32_t)region->offset << ") ...\n";
475 #endif
476  if ((device == nullptr) || (physical_device == nullptr)) {
477  error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
479  }
480  if (block_allocator == nullptr) {
481  error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
483  }
484  return block_allocator->retain(this, region);
485 }
486 
487 bool VulkanMemoryAllocator::collect(void *user_context) {
488 #if defined(HL_VK_DEBUG_MEM)
489  debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
490  << "user_context=" << user_context << ") ... \n";
491 #endif
492  if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
493  return false;
494  }
495  return block_allocator->collect(this);
496 }
497 
498 int VulkanMemoryAllocator::release(void *user_context) {
499 #if defined(HL_VK_DEBUG_MEM)
500  debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
501  << "user_context=" << user_context << ") ... \n";
502 #endif
503  if ((device == nullptr) || (physical_device == nullptr)) {
504  error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
506  }
507  if (block_allocator == nullptr) {
508  error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
510  }
511 
512  return block_allocator->release(this);
513 }
514 
515 int VulkanMemoryAllocator::destroy(void *user_context) {
516 #if defined(HL_VK_DEBUG_MEM)
517  debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
518  << "user_context=" << user_context << ") ... \n";
519 #endif
520  if (block_allocator != nullptr) {
521  block_allocator->destroy(this);
522  }
523  region_count = 0;
524  region_byte_count = 0;
525  block_count = 0;
526  block_byte_count = 0;
528 }
529 
530 const VulkanMemoryConfig &
532  static VulkanMemoryConfig result;
533  return result;
534 }
535 
536 // --
537 int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
538 #if defined(HL_VK_DEBUG_MEM)
539  debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
540  << "user_context=" << user_context << " "
541  << "size=" << (uint32_t)block->size << ", "
542  << "usage_flags=" << usage_flags << ") ... \n";
543 #endif
544  VkBufferCreateInfo create_info = {
546  nullptr, // struct extending this
547  0, // create flags
548  size, // buffer size (in bytes)
549  usage_flags, // buffer usage flags
550  VK_SHARING_MODE_EXCLUSIVE, // sharing mode
551  0, nullptr};
552 
553  // Create a buffer to determine alignment requirements
554  VkBuffer buffer = {0};
555  VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
556  if (result != VK_SUCCESS) {
557 #if defined(HL_VK_DEBUG_MEM)
558  debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
559  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
560 #endif
562  }
563 
564  vkGetBufferMemoryRequirements(this->device, buffer, memory_requirements);
565  vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
567 }
568 
569 int VulkanMemoryAllocator::conform_block_request(void *instance_ptr, MemoryRequest *request) {
570 
571  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
572  if (instance == nullptr) {
574  }
575 
576  void *user_context = instance->owner_context;
577 #if defined(HL_VK_DEBUG_MEM)
578  debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
579  << "user_context=" << user_context << " "
580  << "request=" << (void *)(request) << ") ... \n";
581 #endif
582 
583  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
584  error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
586  }
587 
588  VkMemoryRequirements memory_requirements = {0};
589  uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
590  int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
591  if (error_code != halide_error_code_success) {
592  error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
593  return error_code;
594  }
595 
596 #if defined(HL_VK_DEBUG_MEM)
597  debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
598  << "size=" << (uint32_t)request->size << ", "
599  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
600  << "required_size=" << (uint32_t)memory_requirements.size << ", "
601  << "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
602  << "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
603  << "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
604 #endif
605 
606  request->size = memory_requirements.size;
607  request->properties.alignment = memory_requirements.alignment;
609 }
610 
611 int VulkanMemoryAllocator::allocate_block(void *instance_ptr, MemoryBlock *block) {
612  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
613  if (instance == nullptr) {
615  }
616 
617  void *user_context = instance->owner_context;
618  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
619  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
621  }
622 
623  if (block == nullptr) {
624  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
626  }
627 
628 #if defined(HL_VK_DEBUG_MEM)
629  debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
630  << "user_context=" << user_context << " "
631  << "block=" << (void *)(block) << " "
632  << "size=" << (uint64_t)block->size << ", "
633  << "dedicated=" << (block->dedicated ? "true" : "false") << " "
634  << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
635  << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
636  << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
637 #endif
638 
639  // Find an appropriate memory type given the flags
640  uint32_t memory_type = instance->select_memory_type(user_context, instance->physical_device, block->properties, 0);
641  if (memory_type == invalid_memory_type) {
642  error(user_context) << "VulkanMemoryAllocator: Unable to find appropriate memory type for device!\n";
644  }
645 
646  // Allocate memory
647  VkMemoryAllocateInfo alloc_info = {
649  nullptr, // struct extending this
650  block->size, // size of allocation in bytes
651  memory_type // memory type index from physical device
652  };
653 
654  VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(nullptr, sizeof(VkDeviceMemory), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
655  if (device_memory == nullptr) {
656  debug(nullptr) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
658  }
659 
660  VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
661  if (result != VK_SUCCESS) {
662  debug(nullptr) << "VulkanMemoryAllocator: Allocation failed! vkAllocateMemory returned: " << vk_get_error_name(result) << "\n";
664  }
665 #ifdef DEBUG_RUNTIME
666  debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
667 #endif
668 
669  block->handle = (void *)device_memory;
670  instance->block_byte_count += block->size;
671  instance->block_count++;
673 }
674 
675 int VulkanMemoryAllocator::deallocate_block(void *instance_ptr, MemoryBlock *block) {
676  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
677  if (instance == nullptr) {
679  }
680 
681  void *user_context = instance->owner_context;
682 #if defined(HL_VK_DEBUG_MEM)
683  debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
684  << "user_context=" << user_context << " "
685  << "block=" << (void *)(block) << ") ... \n";
686 #endif
687 
688  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
689  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
691  }
692 
693  if (block == nullptr) {
694  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
696  }
697 
698 #if defined(HL_VK_DEBUG_MEM)
699  debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
700  << "size=" << (uint32_t)block->size << ", "
701  << "dedicated=" << (block->dedicated ? "true" : "false") << " "
702  << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
703  << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
704  << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
705 #endif
706 
707  if (block->handle == nullptr) {
708  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid handle!\n";
710  }
711 
712  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
713  if (device_memory == nullptr) {
714  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
716  }
717 
718  vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
719 #ifdef DEBUG_RUNTIME
720  debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
721 #endif
722 
723  if (instance->block_count > 0) {
724  instance->block_count--;
725  } else {
726  error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
727  instance->block_count = 0;
728  }
729 
730  if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
731  instance->block_byte_count -= block->size;
732  } else {
733  error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
734  instance->block_byte_count = 0;
735  }
736 
737  block->handle = nullptr;
738  vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
739  device_memory = nullptr;
741 }
742 
743 size_t VulkanMemoryAllocator::blocks_allocated() const {
744  return block_count;
745 }
746 
747 size_t VulkanMemoryAllocator::bytes_allocated_for_blocks() const {
748  return block_byte_count;
749 }
750 
751 uint32_t VulkanMemoryAllocator::select_memory_type(void *user_context,
752  VkPhysicalDevice physical_device,
753  MemoryProperties properties,
754  uint32_t required_flags) const {
755 
756  uint32_t want_flags = 0; //< preferred memory flags for requested access type
757  uint32_t need_flags = 0; //< must have in order to enable requested access
758  switch (properties.visibility) {
761  break;
764  break;
768  break;
771  break;
774  default:
775  error(nullptr) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
776  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
777  return invalid_memory_type;
778  };
779 
780  switch (properties.caching) {
782  if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
784  }
785  break;
787  if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
789  }
790  break;
792  if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
794  }
795  break;
798  break;
800  default:
801  error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
802  << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
803  return invalid_memory_type;
804  };
805 
806  VkPhysicalDeviceMemoryProperties device_memory_properties;
807  vkGetPhysicalDeviceMemoryProperties(physical_device, &device_memory_properties);
808 
809  uint32_t result = invalid_memory_type;
810  for (uint32_t i = 0; i < device_memory_properties.memoryTypeCount; ++i) {
811 
812  // if required flags are given, see if the memory type matches the requirement
813  if (required_flags) {
814  if (((required_flags >> i) & 1) == 0) {
815  continue;
816  }
817  }
818 
819  const VkMemoryPropertyFlags properties = device_memory_properties.memoryTypes[i].propertyFlags;
820  if (need_flags) {
821  if ((properties & need_flags) != need_flags) {
822  continue;
823  }
824  }
825 
826  if (want_flags) {
827  if ((properties & want_flags) != want_flags) {
828  continue;
829  }
830  }
831 
832  result = i;
833  break;
834  }
835 
836  if (result == invalid_memory_type) {
837  error(user_context) << "VulkanBlockAllocator: Failed to find appropriate memory type for given properties:\n\t"
838  << "usage=" << halide_memory_usage_name(properties.usage) << " "
839  << "caching=" << halide_memory_caching_name(properties.caching) << " "
840  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
841  return invalid_memory_type;
842  }
843 
844  return result;
845 }
846 
847 // --
848 
849 int VulkanMemoryAllocator::conform(void *user_context, MemoryRequest *request) {
850 
851  // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
852  // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
853  // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
854  // whose size may be required to larger than our requested size (even though we will only ever touch the
855  // size of the region we're managing as within our block)
856 
857  VkMemoryRequirements memory_requirements = {0};
858  uint32_t usage_flags = select_memory_usage(user_context, request->properties);
859  int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
860  if (error_code != halide_error_code_success) {
861  error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
862  return error_code;
863  }
864 
865 #if defined(HL_VK_DEBUG_MEM)
866  debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
867  << "requested_size=" << (uint32_t)region->size << ", "
868  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
869  << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
870 #endif
871 
872  // Enforce any alignment constraints reported by the device limits for each usage type
873  if (usage_flags & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) {
874  if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
875  request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
876  }
877  } else if (usage_flags & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) {
878  if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
879  request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
880  }
881  }
882 
883  // Ensure the request ends on an aligned address
884  if (request->alignment > config.nearest_multiple) {
885  request->properties.nearest_multiple = request->alignment;
886  }
887 
888  size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
889  size_t actual_offset = aligned_offset(request->offset, actual_alignment);
890  size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
891 
892 #if defined(HL_VK_DEBUG_MEM)
893  if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
894  debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
895  << " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
896  << " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
897  << " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
898  << " required.size = " << (uint64_t)memory_requirements.size << ",\n"
899  << " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
900  }
901 #endif
902  request->size = actual_size;
903  request->alignment = actual_alignment;
904  request->offset = actual_offset;
905 
907 }
908 
909 int VulkanMemoryAllocator::conform_region_request(void *instance_ptr, MemoryRequest *request) {
910 
911  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
912  if (instance == nullptr) {
914  }
915 
916  void *user_context = instance->owner_context;
917 #if defined(HL_VK_DEBUG_MEM)
918  debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
919  << "user_context=" << user_context << " "
920  << "request=" << (void *)(region) << ") ... \n";
921 #endif
922 
923  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
924  error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
926  }
927 
928 #if defined(HL_VK_DEBUG_MEM)
929  debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
930  << "size=" << (uint32_t)request->size << ", "
931  << "offset=" << (uint32_t)request->offset << ", "
932  << "dedicated=" << (request->dedicated ? "true" : "false") << " "
933  << "usage=" << halide_memory_usage_name(request->properties.usage) << " "
934  << "caching=" << halide_memory_caching_name(request->properties.caching) << " "
935  << "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
936 #endif
937 
938  return instance->conform(user_context, request);
939 }
940 
941 int VulkanMemoryAllocator::allocate_region(void *instance_ptr, MemoryRegion *region) {
942 
943  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
944  if (instance == nullptr) {
946  }
947 
948  void *user_context = instance->owner_context;
949 #if defined(HL_VK_DEBUG_MEM)
950  debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
951  << "user_context=" << user_context << " "
952  << "region=" << (void *)(region) << ") ... \n";
953 #endif
954 
955  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
956  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
958  }
959 
960  if (region == nullptr) {
961  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
963  }
964 
965 #if defined(HL_VK_DEBUG_MEM)
966  debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
967  << "size=" << (uint32_t)region->size << ", "
968  << "offset=" << (uint32_t)region->offset << ", "
969  << "dedicated=" << (region->dedicated ? "true" : "false") << " "
970  << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
971  << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
972  << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
973 #endif
974 
975  uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
976 
977  VkBufferCreateInfo create_info = {
979  nullptr, // struct extending this
980  0, // create flags
981  region->size, // buffer size (in bytes)
982  usage_flags, // buffer usage flags
983  VK_SHARING_MODE_EXCLUSIVE, // sharing mode
984  0, nullptr};
985 
986  VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
987  if (buffer == nullptr) {
988  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
990  }
991 
992  VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
993  if (result != VK_SUCCESS) {
994  error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
995  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
997  }
998 
999  // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
1000  // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
1001  // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
1002  // whose size may be required to larger than our requested size (even though we will only ever touch the
1003  // size of the region we're managing as within our block)
1004  VkMemoryRequirements memory_requirements = {0};
1005  vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
1006 
1007 #if defined(HL_VK_DEBUG_MEM)
1008  debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1009  << "requested_size=" << (uint32_t)region->size << ", "
1010  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1011  << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1012 #endif
1013 
1014  if (memory_requirements.size > region->size) {
1015  vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1016 #ifdef DEBUG_RUNTIME
1017  debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
1018  << (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
1019 #endif
1020  create_info.size = memory_requirements.size;
1021  VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1022  if (result != VK_SUCCESS) {
1023  error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1024  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1026  }
1027  }
1028 
1029 #ifdef DEBUG_RUNTIME
1030  debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1031 #endif
1032 
1033  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, region);
1034  if (region_allocator == nullptr) {
1035  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1037  }
1038 
1039  BlockResource *block_resource = region_allocator->block_resource();
1040  if (block_resource == nullptr) {
1041  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1043  }
1044 
1045  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
1046  if (device_memory == nullptr) {
1047  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1049  }
1050 
1051  // Finally, bind buffer to the device memory
1052  result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
1053  if (result != VK_SUCCESS) {
1054  error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
1055  << "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
1057  }
1058 
1059  region->handle = (void *)buffer;
1060  region->is_owner = true;
1061  instance->region_byte_count += region->size;
1062  instance->region_count++;
1064 }
1065 
1066 int VulkanMemoryAllocator::deallocate_region(void *instance_ptr, MemoryRegion *region) {
1067  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1068  if (instance == nullptr) {
1070  }
1071 
1072  void *user_context = instance->owner_context;
1073 #if defined(HL_VK_DEBUG_MEM)
1074  debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
1075  << "user_context=" << user_context << " "
1076  << "region=" << (void *)(region) << ") ... \n";
1077 #endif
1078 
1079  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1080  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1082  }
1083 
1084  if (region == nullptr) {
1085  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1087  }
1088 
1089 #if defined(HL_VK_DEBUG_MEM)
1090  debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
1091  << "size=" << (uint32_t)region->size << ", "
1092  << "offset=" << (uint32_t)region->offset << ", "
1093  << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1094  << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1095  << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1096  << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1097 #endif
1098 
1099  if (region->handle == nullptr) {
1100  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1102  }
1103 
1104  VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
1105  if (buffer == nullptr) {
1106  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1108  }
1109 
1110  vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1111 #ifdef DEBUG_RUNTIME
1112  debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1113 #endif
1114  region->handle = nullptr;
1115  if (instance->region_count > 0) {
1116  instance->region_count--;
1117  } else {
1118  error(nullptr) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1119  instance->region_count = 0;
1121  }
1122 
1123  if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
1124  instance->region_byte_count -= region->size;
1125  } else {
1126  error(nullptr) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1127  instance->region_byte_count = 0;
1129  }
1130  vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1131  buffer = nullptr;
1133 }
1134 
1135 size_t VulkanMemoryAllocator::regions_allocated() const {
1136  return region_count;
1137 }
1138 
1139 size_t VulkanMemoryAllocator::bytes_allocated_for_regions() const {
1140  return region_byte_count;
1141 }
1142 
1143 uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
1144  uint32_t result = 0;
1145  switch (properties.usage) {
1148  break;
1152  break;
1155  break;
1158  break;
1161  break;
1164  default:
1165  error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1166  << "usage=" << halide_memory_usage_name(properties.usage) << "\n";
1167  return invalid_usage_flags;
1168  };
1169 
1170  if (result == invalid_usage_flags) {
1171  error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1172  << "usage=" << halide_memory_usage_name(properties.usage) << " "
1173  << "caching=" << halide_memory_caching_name(properties.caching) << " "
1174  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
1175  return invalid_usage_flags;
1176  }
1177 
1178  return result;
1179 }
1180 
1181 // --------------------------------------------------------------------------
1182 
1183 namespace {
1184 
1185 // --------------------------------------------------------------------------
1186 // Halide System allocator for host allocations
1187 void *vk_system_malloc(void *user_context, size_t size) {
1188  return malloc(size);
1189 }
1190 
1191 void vk_system_free(void *user_context, void *ptr) {
1192  free(ptr);
1193 }
1194 
1195 // Vulkan host-side allocation
1196 void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
1197  if (callbacks) {
1198  return callbacks->pfnAllocation(user_context, size, alignment, scope);
1199  } else {
1200  return vk_system_malloc(user_context, size);
1201  }
1202 }
1203 
1204 void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
1205  if (callbacks) {
1206  return callbacks->pfnFree(user_context, ptr);
1207  } else {
1208  return vk_system_free(user_context, ptr);
1209  }
1210 }
1211 
1212 VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
1213  VkDevice device,
1214  VkPhysicalDevice physical_device,
1215  const VkAllocationCallbacks *alloc_callbacks) {
1216 
1217  SystemMemoryAllocatorFns system_allocator = {vk_system_malloc, vk_system_free};
1218  VulkanMemoryConfig config = memory_allocator_config;
1219 
1220  // Parse the allocation config string (if specified).
1221  //
1222  // `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
1223  // allocator use the given constraints specified as three integer values
1224  // separated by a `:` or `;`. These values correspond to `minimum_block_size`,
1225  // `maximum_block_size` and `maximum_block_count`.
1226  //
1227  const char *alloc_config = vk_get_alloc_config_internal(user_context);
1229  StringTable alloc_config_values;
1230  alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
1231  if (alloc_config_values.size() > 0) {
1232  config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
1233  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
1234  }
1235  if (alloc_config_values.size() > 1) {
1236  config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
1237  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
1238  }
1239  if (alloc_config_values.size() > 2) {
1240  config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
1241  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
1242  }
1243  if (alloc_config_values.size() > 3) {
1244  config.maximum_block_count = atoi(alloc_config_values[3]);
1245  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
1246  }
1247  if (alloc_config_values.size() > 4) {
1248  config.nearest_multiple = atoi(alloc_config_values[4]);
1249  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
1250  }
1251  }
1252 
1253  return VulkanMemoryAllocator::create(user_context,
1254  config, device, physical_device,
1255  system_allocator, alloc_callbacks);
1256 }
1257 
1258 int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
1259  if (allocator != nullptr) {
1260  VulkanMemoryAllocator::destroy(user_context, allocator);
1261  allocator = nullptr;
1262  }
1264 }
1265 
1266 // --------------------------------------------------------------------------
1267 
1268 int vk_clear_device_buffer(void *user_context,
1269  VulkanMemoryAllocator *allocator,
1270  VkCommandPool command_pool,
1271  VkQueue command_queue,
1272  VkBuffer device_buffer) {
1273 
1274 #ifdef DEBUG_RUNTIME
1275  debug(user_context)
1276  << " vk_clear_device_buffer (user_context: " << user_context << ", "
1277  << "allocator: " << (void *)allocator << ", "
1278  << "command_pool: " << (void *)command_pool << ", "
1279  << "command_queue: " << (void *)command_queue << ", "
1280  << "device_buffer: " << (void *)device_buffer << ")\n";
1281 #endif
1282 
1283  // create a command buffer
1284  VkCommandBuffer command_buffer;
1285  int error_code = vk_create_command_buffer(user_context, allocator, command_pool, &command_buffer);
1286  if (error_code != halide_error_code_success) {
1287  error(user_context) << "Vulkan: Failed to create command buffer!\n";
1288  return error_code;
1289  }
1290 
1291  // begin the command buffer
1292  VkCommandBufferBeginInfo command_buffer_begin_info =
1293  {
1295  nullptr, // pointer to struct extending this
1297  nullptr // pointer to parent command buffer
1298  };
1299 
1300  VkResult result = vkBeginCommandBuffer(command_buffer, &command_buffer_begin_info);
1301  if (result != VK_SUCCESS) {
1302  error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
1304  }
1305 
1306  // fill buffer with zero values up to the size of the buffer
1307  vkCmdFillBuffer(command_buffer, device_buffer, 0, VK_WHOLE_SIZE, 0);
1308 
1309  // end the command buffer
1310  result = vkEndCommandBuffer(command_buffer);
1311  if (result != VK_SUCCESS) {
1312  error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
1314  }
1315 
1316  // submit the command buffer
1317  VkSubmitInfo submit_info =
1318  {
1319  VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
1320  nullptr, // pointer to struct extending this
1321  0, // wait semaphore count
1322  nullptr, // semaphores
1323  nullptr, // pipeline stages where semaphore waits occur
1324  1, // how many command buffers to execute
1325  &command_buffer, // the command buffers
1326  0, // number of semaphores to signal
1327  nullptr // the semaphores to signal
1328  };
1329 
1330  result = vkQueueSubmit(command_queue, 1, &submit_info, 0);
1331  if (result != VK_SUCCESS) {
1332  error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
1334  }
1335 
1336  // wait for memset to finish
1337  result = vkQueueWaitIdle(command_queue);
1338  if (result != VK_SUCCESS) {
1339  error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
1341  }
1342 
1343  error_code = vk_destroy_command_buffer(user_context, allocator, command_pool, command_buffer);
1344  if (error_code != halide_error_code_success) {
1345  error(user_context) << "Vulkan: Failed to destroy command buffer!\n";
1346  return error_code;
1347  }
1348 
1350 }
1351 
1352 // --------------------------------------------------------------------------
1353 
1354 } // namespace
1355 } // namespace Vulkan
1356 } // namespace Internal
1357 } // namespace Runtime
1358 } // namespace Halide
1359 
1360 // --------------------------------------------------------------------------
1361 
1362 extern "C" {
1363 
1364 // --------------------------------------------------------------------------
1365 
1367  using namespace Halide::Runtime::Internal::Vulkan;
1369  custom_allocation_callbacks = callbacks;
1370 }
1371 
1373  using namespace Halide::Runtime::Internal::Vulkan;
1376 }
1377 
1378 // --------------------------------------------------------------------------
1379 
1380 } // extern "C"
1381 
1382 #endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice device, VkBuffer buffer, VkMemoryRequirements *pMemoryRequirements)
int destroy_crop(void *user_context, MemoryRegion *region)
int release(void *user_context, MemoryRegion *region)
const MemoryAllocators & current_allocators() const
There is a bug in the Halide compiler.
VkDeviceSize minStorageBufferOffsetAlignment
Definition: mini_vulkan.h:1606
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int unmap(void *user_context, MemoryRegion *region)
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)
The Halide runtime encountered an error while trying to allocate memory on device.
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *memory_region)
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo)
#define WEAK
WEAK VulkanMemoryConfig memory_allocator_config
Definition: vulkan_memory.h:32
VkMemoryType memoryTypes[VK_MAX_MEMORY_TYPES]
Definition: mini_vulkan.h:1684
static bool is_empty(const char *str)
An uncategorized error occurred.
VkDeviceSize alignment
Definition: mini_vulkan.h:1753
void * memcpy(void *s1, const void *s2, size_t n)
uint64_t VkDeviceSize
Definition: mini_vulkan.h:71
VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer)
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
int release(void *user_context, MemoryRegion *memory_region)
PFN_vkAllocationFunction pfnAllocation
Definition: mini_vulkan.h:1451
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
void * map(void *user_context, MemoryRegion *region)
void destroy(const T *t)
Because in this header we don&#39;t yet know how client classes store their RefCount (and we don&#39;t want t...
This file defines the class FunctionDAG, which is our representation of a Halide pipeline, and contains methods to using Halide&#39;s bounds tools to query properties of it.
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
Definition: vulkan_memory.h:19
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
#define VK_MAX_MEMORY_TYPES
Definition: mini_vulkan.h:125
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
VkDeviceSize size
Definition: mini_vulkan.h:1864
VkFlags VkMemoryPropertyFlags
Definition: mini_vulkan.h:1126
size_t parse(void *user_context, const char *str, const char *delim)
Definition: string_table.h:159
WEAK const char * halide_memory_usage_name(MemoryUsage value)
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceMemoryProperties *pMemoryProperties)
VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize memoryOffset)
VKAPI_ATTR VkResult VKAPI_CALL vkQueueWaitIdle(VkQueue queue)
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
int retain(void *user_context, MemoryRegion *region)
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceProperties *pProperties)
int atoi(const char *)
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
int reclaim(void *user_context, MemoryRegion *region)
int reclaim(void *user_context, MemoryRegion *region)
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
static int allocate_block(void *instance_ptr, MemoryBlock *block)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
unsigned __INT32_TYPE__ uint32_t
VKAPI_ATTR void VKAPI_CALL vkCmdFillBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDeviceSize dstOffset, VkDeviceSize size, uint32_t data)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
Not visible externally, similar to &#39;static&#39; linkage in C.
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
Definition: vulkan_memory.h:20
signed __INT64_TYPE__ int64_t
A call to halide_malloc returned NULL.
VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence)
static int allocate_region(void *instance_ptr, MemoryRegion *region)
VkSystemAllocationScope
Definition: mini_vulkan.h:365
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMemory)
VkResult
Definition: mini_vulkan.h:138
#define VK_WHOLE_SIZE
Definition: mini_vulkan.h:117
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
VkMemoryPropertyFlags propertyFlags
Definition: mini_vulkan.h:1673
void * malloc(size_t)
VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks *pAllocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
VKAPI_ATTR VkResult VKAPI_CALL vkMapMemory(VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags, void **ppData)
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
#define HL_VK_ENV_DELIM
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
void * memset(void *s, int val, size_t n)
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
VkPhysicalDeviceLimits limits
Definition: mini_vulkan.h:1661
int conform(void *user_context, MemoryRequest *request)
unsigned __INT64_TYPE__ uint64_t
VkDeviceSize minUniformBufferOffsetAlignment
Definition: mini_vulkan.h:1605
VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks *pAllocator)
const VkAllocationCallbacks * callbacks() const
Definition: vulkan_memory.h:82
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
PFN_vkFreeFunction pfnFree
Definition: mini_vulkan.h:1453
VKAPI_ATTR VkResult VKAPI_CALL vkEndCommandBuffer(VkCommandBuffer commandBuffer)
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
Definition: vulkan_memory.h:42
There was no error.
WEAK const char * halide_memory_caching_name(MemoryCaching value)
static void destroy(void *user_context, BlockAllocator *block_allocator)
VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice device, VkDeviceMemory memory)
int retain(void *user_context, MemoryRegion *region)
void free(void *)
static const VulkanMemoryConfig & default_config()