6 #ifndef HALIDE_RUNTIME_BUFFER_H
7 #define HALIDE_RUNTIME_BUFFER_H
20 #include <AvailabilityVersions.h>
21 #include <TargetConditionals.h>
24 #if defined(__has_feature)
25 #if __has_feature(memory_sanitizer)
26 #include <sanitizer/msan_interface.h>
34 #define HALIDE_ALLOCA _alloca
36 #define HALIDE_ALLOCA __builtin_alloca
40 #if __GNUC__ == 5 && __GNUC_MINOR__ == 1
41 #pragma GCC diagnostic ignored "-Warray-bounds"
44 #ifndef HALIDE_RUNTIME_BUFFER_CHECK_INDICES
45 #define HALIDE_RUNTIME_BUFFER_CHECK_INDICES 0
48 #ifndef HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
52 #define HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT 128
56 "HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT must be a power of 2.");
64 #ifndef HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
71 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
73 #elif defined(__ANDROID_API__) && __ANDROID_API__ < 28
76 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
78 #elif defined(__APPLE__)
80 #if TARGET_OS_OSX && (__MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_15)
83 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
85 #elif TARGET_OS_IPHONE && (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_14_0)
88 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
93 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 1
99 #if defined(__GLIBCXX__) && !defined(_GLIBCXX_HAVE_ALIGNED_ALLOC)
102 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
107 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 1
120 template<
typename T,
int Dims,
int InClassDimStorage>
125 template<
typename... Args>
131 template<
typename T,
typename... Args>
133 static const bool value = std::is_convertible<T, int>::value &&
AllInts<Args...>::value;
139 template<
typename... Args>
140 struct AllInts<float, Args...> : std::false_type {};
142 template<
typename... Args>
143 struct AllInts<double, Args...> : std::false_type {};
147 template<
typename Container>
158 static inline void *(*default_allocate_fn)(
size_t) =
nullptr;
218 template<
typename T = void,
237 static const bool T_is_void = std::is_same<typename std::remove_const<T>::type,
void>::value;
240 template<
typename T2>
241 using add_const_if_T_is_const =
typename std::conditional<std::is_const<T>::value,
const T2, T2>::type;
245 using not_void_T =
typename std::conditional<T_is_void,
246 add_const_if_T_is_const<uint8_t>,
250 using not_const_T =
typename std::remove_const<T>::type;
256 using storage_T =
typename std::conditional<std::is_pointer<T>::value,
uint64_t, not_void_T>::type;
260 static constexpr
bool has_static_halide_type = !T_is_void;
265 return halide_type_of<typename std::remove_cv<not_void_T>::type>();
270 return alloc !=
nullptr;
273 static constexpr
bool has_static_dimensions = (Dims !=
AnyDims);
281 static_assert(!has_static_dimensions || static_dimensions() >= 0);
285 void incref()
const {
286 if (owns_host_memory()) {
290 if (!dev_ref_count) {
296 dev_ref_count =
new DeviceRefCount;
298 dev_ref_count->
count++;
304 struct DevRefCountCropped : DeviceRefCount {
311 Buffer<T, AnyDims> cropped_from;
312 explicit DevRefCountCropped(
const Buffer<T, AnyDims> &cropped_from)
313 : cropped_from(cropped_from) {
319 void crop_from(
const Buffer<T, AnyDims> &cropped_from) {
320 assert(dev_ref_count ==
nullptr);
321 dev_ref_count =
new DevRefCountCropped(cropped_from);
326 void decref(
bool device_only =
false) {
327 if (owns_host_memory() && !device_only) {
329 if (new_count == 0) {
331 alloc->~AllocationHeader();
336 set_host_dirty(
false);
340 new_count = --(dev_ref_count->
count);
342 if (new_count == 0) {
344 assert(!(alloc && device_dirty()) &&
345 "Implicitly freeing a dirty device allocation while a host allocation still lives. "
346 "Call device_free explicitly if you want to drop dirty device-side data. "
347 "Call copy_to_host explicitly if you want the data copied to the host allocation "
348 "before the device allocation is freed.");
365 delete (DevRefCountCropped *)dev_ref_count;
367 delete dev_ref_count;
371 dev_ref_count =
nullptr;
376 void free_shape_storage() {
377 if (buf.
dim != shape) {
383 template<
int DimsSpecified>
384 void make_static_shape_storage() {
385 static_assert(Dims ==
AnyDims || Dims == DimsSpecified,
386 "Number of arguments to Buffer() does not match static dimensionality");
388 if constexpr (Dims ==
AnyDims) {
389 if constexpr (DimsSpecified <= InClassDimStorage) {
392 static_assert(DimsSpecified >= 1);
396 static_assert(InClassDimStorage >= Dims);
401 void make_shape_storage(
const int dimensions) {
402 if (Dims !=
AnyDims && Dims != dimensions) {
403 assert(
false &&
"Number of arguments to Buffer() does not match static dimensionality");
417 template<
typename T2,
int D2,
int S2>
418 void move_shape_from(Buffer<T2, D2, S2> &&other) {
419 if (other.shape == other.buf.dim) {
420 copy_shape_from(other.buf);
422 buf.
dim = other.buf.dim;
423 other.buf.dim =
nullptr;
433 dev_ref_count =
new DeviceRefCount;
439 void initialize_shape(
const int *sizes) {
452 void initialize_shape(
const std::vector<int> &sizes) {
454 initialize_shape(sizes.data());
458 template<
typename Array,
size_t N>
459 void initialize_shape_from_array_shape(
int next, Array (&vals)[N]) {
465 initialize_shape_from_array_shape(next - 1, vals[0]);
471 template<
typename T2>
472 void initialize_shape_from_array_shape(
int,
const T2 &) {
476 template<
typename Array,
size_t N>
477 static int dimensionality_of_array(Array (&vals)[N]) {
478 return dimensionality_of_array(vals[0]) + 1;
481 template<
typename T2>
482 static int dimensionality_of_array(
const T2 &) {
487 template<
typename Array,
size_t N>
488 static halide_type_t scalar_type_of_array(Array (&vals)[N]) {
489 return scalar_type_of_array(vals[0]);
492 template<
typename T2>
494 return halide_type_of<typename std::remove_cv<T2>::type>();
498 void crop_host(
int d,
int min,
int extent) {
499 assert(dim(d).
min() <=
min);
500 assert(dim(d).
max() >=
min + extent - 1);
502 if (buf.
host !=
nullptr) {
503 buf.
host += (shift * dim(d).stride()) * type().bytes();
510 void crop_host(
const std::vector<std::pair<int, int>> &rect) {
512 int limit = (int)rect.size();
513 assert(limit <= dimensions());
514 for (
int i = 0; i < limit; i++) {
515 crop_host(i, rect[i].first, rect[i].second);
519 void complete_device_crop(Buffer<T, Dims, InClassDimStorage> &result_host_cropped)
const {
527 result_host_cropped.crop_from(((DevRefCountCropped *)dev_ref_count)->cropped_from);
529 result_host_cropped.crop_from(*
this);
535 void slice_host(
int d,
int pos) {
536 static_assert(Dims ==
AnyDims);
537 assert(dimensions() > 0);
538 assert(d >= 0 && d < dimensions());
539 assert(pos >= dim(d).
min() && pos <= dim(d).
max());
542 if (buf.
host !=
nullptr) {
546 buf.
dim[i] = buf.
dim[i + 1];
551 void complete_device_slice(Buffer<T, AnyDims, InClassDimStorage> &result_host_sliced,
int d,
int pos)
const {
560 result_host_sliced.crop_from(((DevRefCountCropped *)dev_ref_count)->cropped_from);
563 result_host_sliced.crop_from(*
this);
594 return min() + extent() - 1;
605 return val != other.
val;
620 return {
min() + extent()};
630 assert(i >= 0 && i < this->dimensions());
640 return dim(i).extent();
643 return dim(i).stride();
650 return buf.number_of_elements();
655 if constexpr (has_static_dimensions) {
670 assert(buf.
host !=
nullptr);
671 return (T *)buf.begin();
676 assert(buf.
host !=
nullptr);
677 return (T *)buf.end();
682 return buf.size_in_bytes();
694 buf.
type = static_halide_type();
697 constexpr
int buf_dimensions = (Dims ==
AnyDims) ? 0 : Dims;
698 make_static_shape_storage<buf_dimensions>();
704 assert(T_is_void || buf.
type == static_halide_type());
705 initialize_from_buffer(buf, ownership);
709 template<
typename T2,
int D2,
int S2>
713 template<
typename T2,
int D2,
int S2>
714 static void static_assert_can_convert_from() {
715 static_assert((!std::is_const<T2>::value || std::is_const<T>::value),
716 "Can't convert from a Buffer<const T> to a Buffer<T>");
717 static_assert(std::is_same<
typename std::remove_const<T>::type,
718 typename std::remove_const<T2>::type>::value ||
720 "type mismatch constructing Buffer");
722 "Can't convert from a Buffer with static dimensionality to a Buffer with different static dimensionality");
736 template<
typename T2,
int D2,
int S2>
738 static_assert_can_convert_from<T2, D2, S2>();
740 if (other.
type() != static_halide_type()) {
754 template<
typename T2,
int D2,
int S2>
759 static_assert_can_convert_from<T2, D2, S2>();
760 assert(can_convert_from(other));
768 dev_ref_count = other.dev_ref_count;
769 copy_shape_from(other.buf);
778 template<
typename T2,
int D2,
int S2>
782 assert_can_convert_from(other);
784 dev_ref_count = other.dev_ref_count;
785 copy_shape_from(other.buf);
792 dev_ref_count(other.dev_ref_count) {
793 other.dev_ref_count =
nullptr;
794 other.alloc =
nullptr;
802 template<
typename T2,
int D2,
int S2>
806 dev_ref_count(other.dev_ref_count) {
807 assert_can_convert_from(other);
808 other.dev_ref_count =
nullptr;
809 other.alloc =
nullptr;
817 template<
typename T2,
int D2,
int S2>
819 if ((
const void *)
this == (
const void *)&other) {
822 assert_can_convert_from(other);
825 dev_ref_count = other.dev_ref_count;
827 free_shape_storage();
829 copy_shape_from(other.buf);
836 if ((
const void *)
this == (
const void *)&other) {
841 dev_ref_count = other.dev_ref_count;
843 free_shape_storage();
845 copy_shape_from(other.buf);
852 template<
typename T2,
int D2,
int S2>
854 assert_can_convert_from(other);
857 other.alloc =
nullptr;
858 dev_ref_count = other.dev_ref_count;
859 other.dev_ref_count =
nullptr;
860 free_shape_storage();
871 other.alloc =
nullptr;
872 dev_ref_count = other.dev_ref_count;
873 other.dev_ref_count =
nullptr;
874 free_shape_storage();
883 size_t size = type().bytes();
884 for (
int i = 0; i < dimensions(); i++) {
885 size *= dim(i).extent();
888 size = (size << 1) >> 1;
889 for (
int i = 0; i < dimensions(); i++) {
890 size /= dim(i).extent();
892 assert(size == (
size_t)type().bytes() &&
"Error: Overflow computing total size of buffer.");
897 void allocate(
void *(*allocate_fn)(
size_t) =
nullptr,
898 void (*deallocate_fn)(
void *) =
nullptr) {
907 const auto align_up = [=](
size_t value) ->
size_t {
908 return (value + alignment - 1) & ~(alignment - 1);
911 size_t size = size_in_bytes();
913 #if HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
921 void *alloc_storage = ::aligned_alloc(alignment,
align_up(size) + alignment);
935 if (!deallocate_fn) {
937 if (!deallocate_fn) {
938 deallocate_fn =
free;
948 const size_t requested_size =
align_up(size + alignment +
950 (
int)
sizeof(std::max_align_t)));
951 void *alloc_storage = allocate_fn(requested_size);
976 template<
typename... Args,
977 typename =
typename std::enable_if<
AllInts<Args...>::value>::type>
980 assert(static_halide_type() == t);
982 int extents[] = {first, (int)rest...};
984 constexpr
int buf_dimensions = 1 + (int)(
sizeof...(rest));
985 make_static_shape_storage<buf_dimensions>();
986 initialize_shape(extents);
1000 static_assert(!T_is_void,
1001 "To construct an Buffer<void>, pass a halide_type_t as the first argument to the constructor");
1002 int extents[] = {first};
1003 buf.
type = static_halide_type();
1004 constexpr
int buf_dimensions = 1;
1005 make_static_shape_storage<buf_dimensions>();
1006 initialize_shape(extents);
1013 template<
typename... Args,
1014 typename =
typename std::enable_if<
AllInts<Args...>::value>::type>
1015 Buffer(
int first,
int second, Args... rest) {
1016 static_assert(!T_is_void,
1017 "To construct an Buffer<void>, pass a halide_type_t as the first argument to the constructor");
1018 int extents[] = {first, second, (int)rest...};
1019 buf.
type = static_halide_type();
1020 constexpr
int buf_dimensions = 2 + (int)(
sizeof...(rest));
1021 make_static_shape_storage<buf_dimensions>();
1022 initialize_shape(extents);
1033 assert(static_halide_type() == t);
1037 make_shape_storage((
int)sizes.size());
1038 initialize_shape(sizes);
1046 explicit Buffer(
const std::vector<int> &sizes)
1047 :
Buffer(static_halide_type(), sizes) {
1052 static std::vector<int> make_ordered_sizes(
const std::vector<int> &sizes,
const std::vector<int> &order) {
1053 assert(order.size() == sizes.size());
1054 std::vector<int> ordered_sizes(sizes.size());
1055 for (
size_t i = 0; i < sizes.size(); ++i) {
1056 ordered_sizes[i] = sizes.at(order[i]);
1058 return ordered_sizes;
1067 :
Buffer(t, make_ordered_sizes(sizes, storage_order)) {
1068 transpose(storage_order);
1071 Buffer(
const std::vector<int> &sizes,
const std::vector<int> &storage_order)
1072 :
Buffer(static_halide_type(), sizes, storage_order) {
1077 template<
typename Array,
size_t N>
1079 const int buf_dimensions = dimensionality_of_array(vals);
1080 buf.
type = scalar_type_of_array(vals);
1082 make_shape_storage(buf_dimensions);
1083 initialize_shape_from_array_shape(buf.
dimensions - 1, vals);
1090 template<
typename... Args,
1091 typename =
typename std::enable_if<
AllInts<Args...>::value>::type>
1094 assert(static_halide_type() == t);
1096 int extents[] = {first, (int)rest...};
1099 constexpr
int buf_dimensions = 1 + (int)(
sizeof...(rest));
1100 make_static_shape_storage<buf_dimensions>();
1101 initialize_shape(extents);
1107 template<
typename... Args,
1108 typename =
typename std::enable_if<
AllInts<Args...>::value>::type>
1109 explicit Buffer(T *data,
int first, Args &&...rest) {
1110 int extents[] = {first, (int)rest...};
1111 buf.
type = static_halide_type();
1112 buf.
host = (
uint8_t *)
const_cast<typename std::remove_const<T>::type *
>(data);
1113 constexpr
int buf_dimensions = 1 + (int)(
sizeof...(rest));
1114 make_static_shape_storage<buf_dimensions>();
1115 initialize_shape(extents);
1122 explicit Buffer(T *data,
const std::vector<int> &sizes) {
1123 buf.
type = static_halide_type();
1124 buf.
host = (
uint8_t *)
const_cast<typename std::remove_const<T>::type *
>(data);
1125 make_shape_storage((
int)sizes.size());
1126 initialize_shape(sizes);
1135 assert(static_halide_type() == t);
1139 make_shape_storage((
int)sizes.size());
1140 initialize_shape(sizes);
1148 assert(static_halide_type() == t);
1152 make_shape_storage(d);
1153 for (
int i = 0; i < d; i++) {
1154 buf.
dim[i] = shape[i];
1162 const std::vector<halide_dimension_t> &shape)
1163 :
Buffer(t, data, (int)shape.size(), shape.data()) {
1170 buf.
type = static_halide_type();
1171 buf.
host = (
uint8_t *)
const_cast<typename std::remove_const<T>::type *
>(data);
1172 make_shape_storage(d);
1173 for (
int i = 0; i < d; i++) {
1174 buf.
dim[i] = shape[i];
1181 explicit inline Buffer(T *data,
const std::vector<halide_dimension_t> &shape)
1182 :
Buffer(data, (int)shape.size(), shape.data()) {
1190 free_shape_storage();
1217 template<
typename T2,
int D2 = Dims>
1230 template<
typename T2,
int D2 = Dims>
1243 template<
typename T2,
int D2 = Dims>
1272 template<typename T2 = T, typename = typename std::enable_if<!std::is_const<T2>::value>::type>
1279 template<
typename TVoid,
1281 typename =
typename std::enable_if<std::is_same<TVoid, void>::value &&
1282 !std::is_void<T2>::value &&
1283 !std::is_const<T2>::value>::type>
1285 return as<TVoid, Dims>();
1290 template<
typename TVoid,
1292 typename =
typename std::enable_if<std::is_same<TVoid, void>::value &&
1293 !std::is_void<T2>::value &&
1294 std::is_const<T2>::value>::type>
1296 return as<const TVoid, Dims>();
1302 return (dimensions() > 0) ? dim(0).extent() : 1;
1305 return (dimensions() > 1) ? dim(1).extent() : 1;
1308 return (dimensions() > 2) ? dim(2).extent() : 1;
1315 return dim(0).min();
1319 return dim(0).max();
1323 return dim(1).min();
1327 return dim(1).max();
1344 void (*deallocate_fn)(
void *) =
nullptr)
const {
1355 void (*deallocate_fn)(
void *) =
nullptr)
const {
1356 static_assert(Dims ==
AnyDims || Dims == 3);
1357 assert(dimensions() == 3);
1360 dst.
allocate(allocate_fn, deallocate_fn);
1369 void (*deallocate_fn)(
void *) =
nullptr)
const {
1370 std::vector<int> mins, extents;
1371 const int dims = dimensions();
1373 extents.reserve(dims);
1374 for (
int d = 0; d < dims; ++d) {
1375 mins.push_back(dim(d).
min());
1376 extents.push_back(dim(d).extent());
1380 dst.
allocate(allocate_fn, deallocate_fn);
1407 template<
typename T2,
int D2,
int S2>
1409 static_assert(!std::is_const<T>::value,
"Cannot call copy_from() on a Buffer<const T>");
1410 assert(!device_dirty() &&
"Cannot call Halide::Runtime::Buffer::copy_from on a device dirty destination.");
1411 assert(!src.
device_dirty() &&
"Cannot call Halide::Runtime::Buffer::copy_from on a device dirty source.");
1419 const int d = dimensions();
1420 for (
int i = 0; i < d; i++) {
1423 if (max_coord < min_coord) {
1427 dst.
crop(i, min_coord, max_coord - min_coord + 1);
1428 src.
crop(i, min_coord, max_coord - min_coord + 1);
1435 if (T_is_void ? (type().bytes() == 1) : (
sizeof(not_void_T) == 1)) {
1439 typed_dst.
for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1440 }
else if (T_is_void ? (type().bytes() == 2) : (
sizeof(not_void_T) == 2)) {
1444 typed_dst.
for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1445 }
else if (T_is_void ? (type().bytes() == 4) : (
sizeof(not_void_T) == 4)) {
1449 typed_dst.
for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1450 }
else if (T_is_void ? (type().bytes() == 8) : (
sizeof(not_void_T) == 8)) {
1454 typed_dst.
for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1456 assert(
false &&
"type().bytes() must be 1, 2, 4, or 8");
1475 im.crop_host(d,
min, extent);
1477 complete_device_crop(im);
1491 *
this = cropped(d,
min, extent);
1493 crop_host(d,
min, extent);
1513 complete_device_crop(im);
1522 void crop(
const std::vector<std::pair<int, int>> &rect) {
1528 *
this = cropped(rect);
1547 assert(d >= 0 && d < this->dimensions());
1548 device_deallocate();
1563 device_deallocate();
1565 int limit = (int)delta.size();
1566 assert(limit <= dimensions());
1567 for (
int i = 0; i < limit; i++) {
1568 translate(i, delta[i]);
1575 assert(mins.size() <=
static_cast<decltype(mins.size())
>(dimensions()));
1576 device_deallocate();
1577 for (
size_t i = 0; i < mins.size(); i++) {
1578 buf.
dim[i].
min = mins[i];
1582 template<
typename... Args>
1584 set_min(std::vector<int>{args...});
1591 assert(coords.size() <=
static_cast<decltype(coords.size())
>(dimensions()));
1592 for (
size_t i = 0; i < coords.size(); i++) {
1593 if (coords[i] < dim((
int)i).
min() || coords[i] > dim((
int)i).
max()) {
1600 template<
typename... Args>
1602 return contains(std::vector<int>{args...});
1624 assert(d1 >= 0 && d1 < this->dimensions());
1625 assert(d2 >= 0 && d2 < this->dimensions());
1626 std::swap(buf.
dim[d1], buf.
dim[d2]);
1634 assert((
int)order.size() == dimensions());
1635 if (dimensions() < 2) {
1640 std::vector<int> order_sorted = order;
1641 for (
size_t i = 1; i < order_sorted.size(); i++) {
1642 for (
size_t j = i; j > 0 && order_sorted[j - 1] > order_sorted[j]; j--) {
1643 std::swap(order_sorted[j], order_sorted[j - 1]);
1644 transpose(j, j - 1);
1661 static_assert(Dims ==
AnyDims || Dims > 0,
"Cannot slice a 0-dimensional buffer");
1662 assert(dimensions() > 0);
1671 im.slice_host(d, pos);
1673 complete_device_slice(im, d, pos);
1682 static_assert(Dims ==
AnyDims || Dims > 0,
"Cannot slice a 0-dimensional buffer");
1683 assert(dimensions() > 0);
1685 return sliced(d, dim(d).
min());
1694 static_assert(Dims ==
AnyDims,
"Cannot call slice() on a Buffer with static dimensionality.");
1695 assert(dimensions() > 0);
1702 *
this = sliced(d, pos);
1733 static_assert(Dims ==
AnyDims,
"Cannot call embed() on a Buffer with static dimensionality.");
1734 assert(d >= 0 && d <= dimensions());
1736 translate(dimensions() - 1, pos);
1737 for (
int i = dimensions() - 1; i > d; i--) {
1738 transpose(i, i - 1);
1747 static_assert(Dims ==
AnyDims,
"Cannot call add_dimension() on a Buffer with static dimensionality.");
1750 if (buf.
dim != shape) {
1753 for (
int i = 0; i < dims; i++) {
1754 new_shape[i] = buf.
dim[i];
1757 buf.
dim = new_shape;
1758 }
else if (dims == InClassDimStorage) {
1761 for (
int i = 0; i < dims; i++) {
1762 buf.
dim[i] = shape[i];
1767 buf.
dim[dims] = {0, 1, 0};
1789 assert((!v || !device_dirty()) &&
"Cannot set host dirty when device is already dirty. Call copy_to_host() before accessing the buffer from host.");
1790 buf.set_host_dirty(v);
1798 return buf.device_dirty();
1802 return buf.host_dirty();
1806 assert((!v || !host_dirty()) &&
"Cannot set device dirty when host is already dirty.");
1807 buf.set_device_dirty(v);
1811 if (device_dirty()) {
1819 return device_interface->
copy_to_device(ctx, &buf, device_interface);
1825 return device_interface->
device_malloc(ctx, &buf, device_interface);
1829 if (dev_ref_count) {
1831 "Can't call device_free on an unmanaged or wrapped native device handle. "
1832 "Free the source allocation or call device_detach_native instead.");
1834 assert(dev_ref_count->
count == 1 &&
1835 "Multiple Halide::Runtime::Buffer objects share this device "
1836 "allocation. Freeing it would create dangling references. "
1837 "Don't call device_free on Halide buffers that you have copied or "
1838 "passed by value.");
1844 if (dev_ref_count) {
1845 delete dev_ref_count;
1846 dev_ref_count =
nullptr;
1852 uint64_t handle,
void *ctx =
nullptr) {
1853 assert(device_interface);
1856 return device_interface->
wrap_native(ctx, &buf, handle, device_interface);
1860 assert(dev_ref_count &&
1862 "Only call device_detach_native on buffers wrapping a native "
1863 "device handle via device_wrap_native. This buffer was allocated "
1864 "using device_malloc, or is unmanaged. "
1865 "Call device_free or free the original allocation instead.");
1867 assert(dev_ref_count->
count == 1 &&
1868 "Multiple Halide::Runtime::Buffer objects share this device "
1869 "allocation. Freeing it could create dangling references. "
1870 "Don't call device_detach_native on Halide buffers that you "
1871 "have copied or passed by value.");
1876 delete dev_ref_count;
1877 dev_ref_count =
nullptr;
1886 if (dev_ref_count) {
1888 "Can't call device_and_host_free on a device handle not allocated with device_and_host_malloc. "
1889 "Free the source allocation or call device_detach_native instead.");
1891 assert(dev_ref_count->
count == 1 &&
1892 "Multiple Halide::Runtime::Buffer objects share this device "
1893 "allocation. Freeing it would create dangling references. "
1894 "Don't call device_and_host_free on Halide buffers that you have copied or "
1895 "passed by value.");
1901 if (dev_ref_count) {
1902 delete dev_ref_count;
1903 dev_ref_count =
nullptr;
1909 return buf.device_sync(ctx);
1918 if (dev_ref_count ==
nullptr) {
1932 static_assert(Dims ==
AnyDims || Dims == 3,
"make_interleaved() must be called on a Buffer that can represent 3 dimensions.");
1948 return make_interleaved(static_halide_type(), width, height, channels);
1954 static_assert(Dims ==
AnyDims || Dims == 3,
"make_interleaved() must be called on a Buffer that can represent 3 dimensions.");
1963 return make_interleaved(static_halide_type(), data, width, height, channels);
1968 static_assert(Dims ==
AnyDims || Dims == 0,
"make_scalar() must be called on a Buffer that can represent 0 dimensions.");
1976 static_assert(Dims ==
AnyDims || Dims == 0,
"make_scalar() must be called on a Buffer that can represent 0 dimensions.");
1984 static_assert(Dims ==
AnyDims || Dims == 0,
"make_scalar() must be called on a Buffer that can represent 0 dimensions.");
1992 template<
typename T2,
int D2,
int S2>
1994 void *(*allocate_fn)(
size_t) =
nullptr,
1995 void (*deallocate_fn)(
void *) =
nullptr) {
1996 static_assert(Dims == D2 || Dims ==
AnyDims);
1997 const halide_type_t dst_type = T_is_void ? src.
type() : halide_type_of<typename std::remove_cv<not_void_T>::type>();
1999 allocate_fn, deallocate_fn);
2006 void *(*allocate_fn)(
size_t),
2007 void (*deallocate_fn)(
void *)) {
2009 std::vector<int> swaps;
2010 for (
int i = dimensions - 1; i > 0; i--) {
2011 for (
int j = i; j > 0; j--) {
2012 if (shape[j - 1].stride > shape[j].stride) {
2013 std::swap(shape[j - 1], shape[j]);
2021 for (
int i = 0; i < dimensions; i++) {
2030 while (!swaps.empty()) {
2031 int j = swaps.back();
2032 std::swap(shape[j - 1], shape[j]);
2038 Buffer<> dst(dst_type,
nullptr, dimensions, shape);
2039 dst.allocate(allocate_fn, deallocate_fn);
2044 template<
typename... Args>
2047 offset_of(
int d,
int first, Args... rest)
const {
2048 #if HALIDE_RUNTIME_BUFFER_CHECK_INDICES
2049 assert(first >= this->buf.
dim[d].
min);
2050 assert(first < this->buf.
dim[d].
min + this->buf.dim[d].extent);
2052 return offset_of(d + 1, rest...) + (
ptrdiff_t)this->buf.
dim[d].
stride * (first - this->buf.dim[d].min);
2060 template<
typename... Args>
2063 address_of(Args... args)
const {
2065 return (storage_T *)(this->buf.
host) + offset_of(0, args...) * type().bytes();
2067 return (storage_T *)(this->buf.
host) + offset_of(0, args...);
2072 ptrdiff_t offset_of(
const int *pos)
const {
2074 for (
int i = this->dimensions() - 1; i >= 0; i--) {
2075 #if HALIDE_RUNTIME_BUFFER_CHECK_INDICES
2076 assert(pos[i] >= this->buf.
dim[i].
min);
2077 assert(pos[i] < this->buf.
dim[i].
min + this->buf.dim[i].extent);
2085 storage_T *address_of(
const int *pos)
const {
2087 return (storage_T *)this->buf.
host + offset_of(pos) * type().bytes();
2089 return (storage_T *)this->buf.
host + offset_of(pos);
2096 return (T *)(this->buf.
host);
2106 template<
typename... Args,
2107 typename =
typename std::enable_if<
AllInts<Args...>::value>::type>
2109 static_assert(!T_is_void,
2110 "Cannot use operator() on Buffer<void> types");
2111 constexpr
int expected_dims = 1 + (int)(
sizeof...(rest));
2112 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2113 assert(!device_dirty());
2114 return *((
const not_void_T *)(address_of(first, rest...)));
2120 static_assert(!T_is_void,
2121 "Cannot use operator() on Buffer<void> types");
2122 constexpr
int expected_dims = 0;
2123 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2124 assert(!device_dirty());
2125 return *((
const not_void_T *)(data()));
2131 static_assert(!T_is_void,
2132 "Cannot use operator() on Buffer<void> types");
2133 assert(!device_dirty());
2134 return *((
const not_void_T *)(address_of(pos)));
2137 template<
typename... Args,
2138 typename =
typename std::enable_if<
AllInts<Args...>::value>::type>
2142 static_assert(!T_is_void,
2143 "Cannot use operator() on Buffer<void> types");
2144 constexpr
int expected_dims = 1 + (int)(
sizeof...(rest));
2145 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2147 return *((not_void_T *)(address_of(first, rest...)));
2153 static_assert(!T_is_void,
2154 "Cannot use operator() on Buffer<void> types");
2155 constexpr
int expected_dims = 0;
2156 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2158 return *((not_void_T *)(data()));
2164 static_assert(!T_is_void,
2165 "Cannot use operator() on Buffer<void> types");
2167 return *((not_void_T *)(address_of(pos)));
2173 bool all_equal =
true;
2174 for_each_element([&](
const int *pos) { all_equal &= (*this)(pos) == val; });
2180 for_each_value([=](T &v) { v = val; });
2188 struct for_each_value_task_dim {
2196 template<
typename Ptr,
typename... Ptrs>
2199 advance_ptrs(stride + 1, ptrs...);
2206 template<
typename Fn,
typename Ptr,
typename... Ptrs>
2207 HALIDE_NEVER_INLINE static void for_each_value_helper(Fn &&f,
int d,
bool innermost_strides_are_one,
2208 const for_each_value_task_dim<
sizeof...(Ptrs) + 1> *t, Ptr ptr, Ptrs... ptrs) {
2210 if (innermost_strides_are_one) {
2211 Ptr end = ptr + t[0].extent;
2212 while (ptr != end) {
2213 f(*ptr++, (*ptrs++)...);
2217 f(*ptr, (*ptrs)...);
2218 advance_ptrs(t[0].stride, ptr, ptrs...);
2223 for_each_value_helper(f, d - 1, innermost_strides_are_one, t, ptr, ptrs...);
2224 advance_ptrs(t[d].stride, ptr, ptrs...);
2231 HALIDE_NEVER_INLINE static std::pair<int, bool> for_each_value_prep(for_each_value_task_dim<N> *t,
2233 const int dimensions = buffers[0]->
dimensions;
2234 assert(dimensions > 0);
2237 for (
int i = 0; i < N; i++) {
2238 if (buffers[i]->device) {
2239 assert(buffers[i]->host &&
2240 "Buffer passed to for_each_value has device allocation but no host allocation. Call allocate() and copy_to_host() first");
2241 assert(!buffers[i]->device_dirty() &&
2242 "Buffer passed to for_each_value is dirty on device. Call copy_to_host() first");
2244 assert(buffers[i]->host &&
2245 "Buffer passed to for_each_value has no host or device allocation");
2250 for (
int i = 0; i < dimensions; i++) {
2251 for (
int j = 0; j < N; j++) {
2252 assert(buffers[j]->dimensions == dimensions);
2253 assert(buffers[j]->dim[i].extent == buffers[0]->dim[i].extent &&
2254 buffers[j]->dim[i].
min == buffers[0]->dim[i].
min);
2255 const int s = buffers[j]->
dim[i].
stride;
2258 t[i].extent = buffers[0]->
dim[i].
extent;
2263 for (
int j = i; j > 0 && t[j].stride[N - 1] < t[j - 1].stride[N - 1]; j--) {
2264 std::swap(t[j], t[j - 1]);
2271 for (
int i = 1; i < d; i++) {
2273 for (
int j = 0; j < N; j++) {
2274 flat = flat && t[i - 1].stride[j] * t[i - 1].extent == t[i].stride[j];
2277 t[i - 1].extent *= t[i].extent;
2278 for (
int j = i; j < d - 1; j++) {
2289 bool innermost_strides_are_one =
true;
2290 for (
int i = 0; i < N; i++) {
2291 innermost_strides_are_one &= (t[0].stride[i] == 1);
2294 return {d, innermost_strides_are_one};
2297 template<
typename Fn,
typename... Args,
int N =
sizeof...(Args) + 1>
2298 void for_each_value_impl(Fn &&f, Args &&...other_buffers)
const {
2299 if (dimensions() > 0) {
2300 const size_t alloc_size = dimensions() *
sizeof(for_each_value_task_dim<N>);
2301 Buffer<>::for_each_value_task_dim<N> *t =
2302 (Buffer<>::for_each_value_task_dim<N> *)
HALIDE_ALLOCA(alloc_size);
2306 auto [new_dims, innermost_strides_are_one] = Buffer<>::for_each_value_prep(t, buffers);
2308 Buffer<>::for_each_value_helper(f, new_dims - 1,
2309 innermost_strides_are_one,
2311 data(), (other_buffers.data())...);
2318 f(*data(), (*other_buffers.data())...);
2338 template<
typename Fn,
typename... Args,
int N =
sizeof...(Args) + 1>
2340 for_each_value_impl(f, std::forward<Args>(other_buffers)...);
2344 template<
typename Fn,
typename... Args,
int N =
sizeof...(Args) + 1>
2348 for_each_value_impl(f, std::forward<Args>(other_buffers)...);
2355 struct for_each_element_task_dim {
2362 template<
typename Fn,
2364 typename = decltype(std::declval<Fn>()(std::declval<Args>()...))>
2365 HALIDE_ALWAYS_INLINE static void for_each_element_variadic(
int,
int,
const for_each_element_task_dim *, Fn &&f, Args... args) {
2371 template<
typename Fn,
2373 HALIDE_ALWAYS_INLINE static void for_each_element_variadic(
double,
int d,
const for_each_element_task_dim *t, Fn &&f, Args... args) {
2374 for (
int i = t[d].
min; i <= t[d].max; i++) {
2375 for_each_element_variadic(0, d - 1, t, std::forward<Fn>(f), i, args...);
2381 template<
typename Fn,
2383 typename = decltype(std::declval<Fn>()(std::declval<Args>()...))>
2385 return (
int)(
sizeof...(Args));
2391 template<
typename Fn,
2394 static_assert(
sizeof...(args) <= 256,
2395 "Callable passed to for_each_element must accept either a const int *,"
2396 " or up to 256 ints. No such operator found. Expect infinite template recursion.");
2397 return num_args(0, std::forward<Fn>(f), 0, args...);
2407 typename =
typename std::enable_if<(d >= 0)>::type>
2408 HALIDE_ALWAYS_INLINE static void for_each_element_array_helper(
int,
const for_each_element_task_dim *t, Fn &&f,
int *pos) {
2409 for (pos[d] = t[d].
min; pos[d] <= t[d].max; pos[d]++) {
2410 for_each_element_array_helper<d - 1>(0, t, std::forward<Fn>(f), pos);
2417 typename =
typename std::enable_if<(d < 0)>::type>
2418 HALIDE_ALWAYS_INLINE static void for_each_element_array_helper(
double,
const for_each_element_task_dim *t, Fn &&f,
int *pos) {
2427 template<
typename Fn>
2428 static void for_each_element_array(
int d,
const for_each_element_task_dim *t, Fn &&f,
int *pos) {
2431 }
else if (d == 0) {
2435 for_each_element_array_helper<0, Fn>(0, t, std::forward<Fn>(f), pos);
2436 }
else if (d == 1) {
2437 for_each_element_array_helper<1, Fn>(0, t, std::forward<Fn>(f), pos);
2438 }
else if (d == 2) {
2439 for_each_element_array_helper<2, Fn>(0, t, std::forward<Fn>(f), pos);
2440 }
else if (d == 3) {
2441 for_each_element_array_helper<3, Fn>(0, t, std::forward<Fn>(f), pos);
2443 for (pos[d] = t[d].
min; pos[d] <= t[d].max; pos[d]++) {
2444 for_each_element_array(d - 1, t, std::forward<Fn>(f), pos);
2452 template<
typename Fn,
2453 typename = decltype(std::declval<Fn>()((
const int *)
nullptr))>
2454 static void for_each_element(
int,
int dims,
const for_each_element_task_dim *t, Fn &&f,
int check = 0) {
2455 const int size = dims *
sizeof(int);
2460 for_each_element_array(dims - 1, t, std::forward<Fn>(f), pos);
2465 template<
typename Fn>
2466 HALIDE_ALWAYS_INLINE static void for_each_element(
double,
int dims,
const for_each_element_task_dim *t, Fn &&f) {
2467 int args = num_args(0, std::forward<Fn>(f));
2468 assert(dims >= args);
2469 for_each_element_variadic(0, args - 1, t, std::forward<Fn>(f));
2472 template<
typename Fn>
2473 void for_each_element_impl(Fn &&f)
const {
2474 for_each_element_task_dim *t =
2475 (for_each_element_task_dim *)
HALIDE_ALLOCA(dimensions() *
sizeof(for_each_element_task_dim));
2476 for (
int i = 0; i < dimensions(); i++) {
2477 t[i].min = dim(i).min();
2478 t[i].max = dim(i).max();
2480 for_each_element(0, dimensions(), t, std::forward<Fn>(f));
2541 template<
typename Fn>
2543 for_each_element_impl(f);
2547 template<
typename Fn>
2551 for_each_element_impl(f);
2557 template<
typename Fn>
2562 template<
typename... Args,
2563 typename = decltype(std::declval<Fn>()(std::declval<Args>()...))>
2564 void operator()(Args... args) {
2565 (*buf)(args...) = f(args...);
2569 : f(std::forward<Fn>(f)), buf(buf) {
2578 template<
typename Fn,
2579 typename =
typename std::enable_if<!std::is_arithmetic<typename std::decay<Fn>::type>::value>::type>
2582 FillHelper<Fn> wrapper(std::forward<Fn>(f),
this);
2583 return for_each_element(wrapper);
2600 #if defined(__has_feature)
2601 #if __has_feature(memory_sanitizer)
2603 __msan_check_mem_is_initialized(data(), size_in_bytes());
2605 for_each_value([](T &v) { __msan_check_mem_is_initialized(&v,
sizeof(T)); ; });
2615 #undef HALIDE_ALLOCA
#define HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
This file declares the routines used by Halide internally in its runtime.
#define HALIDE_NEVER_INLINE
@ halide_error_code_success
There was no error.
#define HALIDE_ALWAYS_INLINE
struct halide_buffer_t halide_buffer_t
The raw representation of an image passed around by generated Halide code.
A Halide::Buffer is a named shared reference to a Halide::Runtime::Buffer.
Read-only access to the shape.
HALIDE_ALWAYS_INLINE int min() const
The lowest coordinate in this dimension.
Dimension(const halide_dimension_t &dim)
HALIDE_ALWAYS_INLINE int max() const
The highest coordinate in this dimension.
HALIDE_ALWAYS_INLINE iterator end() const
An iterator that points to one past the max coordinate.
HALIDE_ALWAYS_INLINE int stride() const
The number of elements in memory you have to step over to increment this coordinate by one.
HALIDE_ALWAYS_INLINE iterator begin() const
An iterator that points to the min coordinate.
HALIDE_ALWAYS_INLINE int extent() const
The extent of the image along this dimension.
A templated Buffer class that wraps halide_buffer_t and adds functionality.
Buffer< T,(Dims==AnyDims ? AnyDims :Dims+1)> embedded(int d, int pos=0) const
Make a new buffer that views this buffer as a single slice in a higher-dimensional space.
void translate(int d, int delta)
Translate an image in-place along one dimension by changing how it is indexed.
Buffer(const halide_buffer_t &buf, BufferDeviceOwnership ownership=BufferDeviceOwnership::Unmanaged)
Make a Buffer from a halide_buffer_t.
void allocate(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr)
Allocate memory for this Buffer.
void add_dimension()
Add a new dimension with a min of zero and an extent of one.
T * end() const
A pointer to one beyond the element with the highest address.
void slice(int d)
Slice a buffer in-place at the dimension's minimum.
static void set_default_allocate_fn(void *(*allocate_fn)(size_t))
bool owns_host_memory() const
Does this Buffer own the host memory it refers to?
HALIDE_ALWAYS_INLINE not_void_T & operator()(int first, Args... rest)
int width() const
Conventional names for the first three dimensions.
Buffer< T, Dims, InClassDimStorage > translated(int d, int dx) const
Make an image which refers to the same data with using translated coordinates in the given dimension.
HALIDE_ALWAYS_INLINE const Buffer< T2, D2, InClassDimStorage > & as() const &
Return a const typed reference to this Buffer.
void transpose(const std::vector< int > &order)
A generalized transpose: instead of swapping two dimensions, pass a vector that lists each dimension ...
void set_min(const std::vector< int > &mins)
Set the min coordinate of an image in the first N dimensions.
Buffer< T,(Dims==AnyDims ? AnyDims :Dims - 1)> sliced(int d, int pos) const
Make a lower-dimensional buffer that refers to one slice of this buffer.
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, const std::vector< int > &sizes)
Initialize an Buffer of runtime type from a pointer and a vector of sizes.
static Buffer< T, Dims, InClassDimStorage > make_interleaved(int width, int height, int channels)
If you use the (x, y, c) indexing convention, then Halide Buffers are stored planar by default.
int copy_to_host(void *ctx=nullptr)
Buffer(halide_type_t t, const std::vector< int > &sizes)
Allocate a new image of unknown type using a vector of ints as the size.
int device_malloc(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
int device_free(void *ctx=nullptr)
bool contains(Args... args) const
HALIDE_ALWAYS_INLINE Buffer< T, Dims, InClassDimStorage > & for_each_value(Fn &&f, Args &&...other_buffers)
void set_device_dirty(bool v=true)
Buffer(T *data, int d, const halide_dimension_t *shape)
Initialize an Buffer from a pointer to the min coordinate and an array describing the shape.
Buffer(Buffer< T2, D2, S2 > &&other)
Move-construct a Buffer from a Buffer of different dimensionality and type.
void slice(int d, int pos)
Rewrite the buffer to refer to a single lower-dimensional slice of itself along the given dimension a...
HALIDE_ALWAYS_INLINE void set_host_dirty(bool v=true)
Methods for managing any GPU allocation.
void msan_check_mem_is_initialized(bool entire=false) const
Convenient check to verify that all of the interesting bytes in the Buffer are initialized under MSAN...
HALIDE_ALWAYS_INLINE Buffer< T2, D2, InClassDimStorage > as() &&
Return an rval reference to this Buffer.
Buffer< T, Dims, InClassDimStorage > & operator=(const Buffer< T2, D2, S2 > &other)
Assign from another Buffer of possibly-different dimensionality and type.
static Buffer< T, Dims, InClassDimStorage > make_scalar()
Make a zero-dimensional Buffer.
int device_detach_native(void *ctx=nullptr)
Buffer< T, Dims, InClassDimStorage > translated(const std::vector< int > &delta) const
Make an image which refers to the same data translated along the first N dimensions.
int device_wrap_native(const struct halide_device_interface_t *device_interface, uint64_t handle, void *ctx=nullptr)
HALIDE_ALWAYS_INLINE Dimension dim(int i) const
Access the shape of the buffer.
HALIDE_ALWAYS_INLINE Buffer< T, Dims, InClassDimStorage > & for_each_element(Fn &&f)
Buffer< T, Dims, InClassDimStorage > cropped(int d, int min, int extent) const
Make an image that refers to a sub-range of this image along the given dimension.
HALIDE_ALWAYS_INLINE Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > as_const() &&
Buffer(int first, int second, Args... rest)
HALIDE_ALWAYS_INLINE const Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > & as_const() const &
BufferDeviceOwnership device_ownership() const
Return the method by which the device field is managed.
void check_overflow()
Check the product of the extents fits in memory.
static bool can_convert_from(const Buffer< T2, D2, S2 > &other)
Determine if a Buffer<T, Dims, InClassDimStorage> can be constructed from some other Buffer type.
int device_and_host_malloc(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
int device_sync(void *ctx=nullptr)
HALIDE_ALWAYS_INLINE const not_void_T & operator()() const
Buffer(const std::vector< int > &sizes)
Allocate a new image of known type using a vector of ints as the size.
void embed(int d, int pos=0)
Embed a buffer in-place, increasing the dimensionality.
static constexpr halide_type_t static_halide_type()
Get the Halide type of T.
Buffer(T *data, int first, Args &&...rest)
Initialize an Buffer from a pointer and some sizes.
HALIDE_ALWAYS_INLINE not_void_T & operator()(const int *pos)
int copy_to_device(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Buffer< T, Dims, InClassDimStorage > cropped(const std::vector< std::pair< int, int >> &rect) const
Make an image that refers to a sub-rectangle of this image along the first N dimensions.
Buffer(Array(&vals)[N])
Make an Buffer that refers to a statically sized array.
static Buffer< void, Dims, InClassDimStorage > make_interleaved(halide_type_t t, int width, int height, int channels)
If you use the (x, y, c) indexing convention, then Halide Buffers are stored planar by default.
static Buffer< add_const_if_T_is_const< void >, Dims, InClassDimStorage > make_interleaved(halide_type_t t, T *data, int width, int height, int channels)
Wrap an existing interleaved image.
halide_type_t type() const
Get the type of the elements.
int device_and_host_free(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Buffer< T, Dims, InClassDimStorage > & fill(Fn &&f)
Fill a buffer by evaluating a callable at every site.
Buffer(int first)
Allocate a new image of the given size.
Buffer< not_const_T, Dims, InClassDimStorage > copy_to_interleaved(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Like copy(), but the copy is created in interleaved memory layout (vs.
HALIDE_ALWAYS_INLINE bool device_dirty() const
static Buffer< T, Dims, InClassDimStorage > make_scalar(T *data)
Make a zero-dimensional Buffer that points to non-owned, existing data.
static constexpr int static_dimensions()
Callers should not use the result if has_static_dimensions is false.
Buffer< not_const_T, Dims, InClassDimStorage > copy_to_planar(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Like copy(), but the copy is created in planar memory layout (vs.
void transpose(int d1, int d2)
Transpose a buffer in-place by changing how it is indexed.
HALIDE_ALWAYS_INLINE const not_void_T & operator()(const int *pos) const
void deallocate()
Drop reference to any owned host or device memory, possibly freeing it, if this buffer held the last ...
size_t size_in_bytes() const
The total number of bytes spanned by the data in memory.
HALIDE_ALWAYS_INLINE const Buffer< T, Dims, InClassDimStorage > & for_each_value(Fn &&f, Args &&...other_buffers) const
Call a function on every value in the buffer, and the corresponding values in some number of other bu...
bool has_device_allocation() const
halide_buffer_t * raw_buffer()
Get a pointer to the raw halide_buffer_t this wraps.
Buffer< T, Dims, InClassDimStorage > & operator=(Buffer< T, Dims, InClassDimStorage > &&other) noexcept
Standard move-assignment operator.
Buffer< T, Dims, InClassDimStorage > transposed(const std::vector< int > &order) const
Make a buffer which refers to the same data in the same layout using a different ordering of the dime...
static Buffer< T, Dims, InClassDimStorage > make_with_shape_of(Buffer< T2, D2, S2 > src, void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr)
Make a buffer with the same shape and memory nesting order as another buffer.
void reset()
Reset the Buffer to be equivalent to a default-constructed Buffer of the same static type (if any); B...
HALIDE_ALWAYS_INLINE Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > & as_const() &
as_const() is syntactic sugar for .as<const T>(), to avoid the need to recapitulate the type argument...
HALIDE_ALWAYS_INLINE const not_void_T & operator()(int first, Args... rest) const
Access elements.
Buffer(halide_type_t t, int first, Args... rest)
Allocate a new image of the given size with a runtime type.
int dimensions() const
Get the dimensionality of the buffer.
Buffer< T, Dims, InClassDimStorage > & operator=(const Buffer< T, Dims, InClassDimStorage > &other)
Standard assignment operator.
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, int d, const halide_dimension_t *shape)
Initialize an Buffer from a pointer to the min coordinate and an array describing the shape.
int min(int i) const
Access to the mins, strides, extents.
Buffer< T, Dims, InClassDimStorage > & operator=(Buffer< T2, D2, S2 > &&other)
Move from another Buffer of possibly-different dimensionality and type.
void device_deallocate()
Drop reference to any owned device memory, possibly freeing it if this buffer held the last reference...
void add_dimension_with_stride(int s)
Add a new dimension with a min of zero, an extent of one, and the specified stride.
Buffer< T, Dims, InClassDimStorage > & fill(not_void_T val)
Buffer(Buffer< T, Dims, InClassDimStorage > &&other) noexcept
Move constructor.
static Buffer< add_const_if_T_is_const< void >, Dims, InClassDimStorage > make_scalar(halide_type_t t)
Make a zero-dimensional Buffer.
HALIDE_ALWAYS_INLINE not_void_T & operator()()
void crop(const std::vector< std::pair< int, int >> &rect)
Crop an image in-place along the first N dimensions.
void crop(int d, int min, int extent)
Crop an image in-place along the given dimension.
void set_min(Args... args)
Buffer< T, Dims, InClassDimStorage > transposed(int d1, int d2) const
Make a buffer which refers to the same data in the same layout using a swapped indexing order for the...
Buffer< T,(Dims==AnyDims ? AnyDims :Dims - 1)> sliced(int d) const
Make a lower-dimensional buffer that refers to one slice of this buffer at the dimension's minimum.
size_t number_of_elements() const
The total number of elements this buffer represents.
static void assert_can_convert_from(const Buffer< T2, D2, S2 > &other)
Fail an assertion at runtime or compile-time if an Buffer<T, Dims, InClassDimStorage> cannot be const...
void translate(const std::vector< int > &delta)
Translate an image along the first N dimensions by changing how it is indexed.
Buffer(const Buffer< T, Dims, InClassDimStorage > &other)
Copy constructor.
Buffer(const std::vector< int > &sizes, const std::vector< int > &storage_order)
Buffer(halide_type_t t, const std::vector< int > &sizes, const std::vector< int > &storage_order)
Allocate a new image of unknown type using a vector of ints as the size and a vector of indices indic...
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, const std::vector< halide_dimension_t > &shape)
Initialize a Buffer from a pointer to the min coordinate and a vector describing the shape.
T * data() const
Get a pointer to the address of the min coordinate.
bool is_bounds_query() const
Check if an input buffer passed extern stage is a querying bounds.
int left() const
Conventional names for the min and max value of each dimension.
void copy_from(Buffer< T2, D2, S2 > src)
Fill a Buffer with the values at the same coordinates in another Buffer.
static Buffer< T, Dims, InClassDimStorage > make_interleaved(T *data, int width, int height, int channels)
Wrap an existing interleaved image.
Buffer< not_const_T, Dims, InClassDimStorage > copy(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Make a new image which is a deep copy of this image.
Buffer< T, Dims, InClassDimStorage > alias() const
Make a copy of the Buffer which shares the underlying host and/or device allocations as the existing ...
static void set_default_deallocate_fn(void(*deallocate_fn)(void *))
HALIDE_ALWAYS_INLINE const Buffer< T, Dims, InClassDimStorage > & for_each_element(Fn &&f) const
Call a function at each site in a buffer.
Buffer(const Buffer< T2, D2, S2 > &other)
Construct a Buffer from a Buffer of different dimensionality and type.
bool contains(const std::vector< int > &coords) const
Test if a given coordinate is within the bounds of an image.
T * begin() const
A pointer to the element with the lowest address.
Buffer(T *data, const std::vector< halide_dimension_t > &shape)
Initialize a Buffer from a pointer to the min coordinate and a vector describing the shape.
Buffer(T *data, const std::vector< int > &sizes)
Initialize an Buffer from a pointer and a vector of sizes.
bool all_equal(not_void_T val) const
Tests that all values in this buffer are equal to val.
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, int first, Args &&...rest)
Initialize an Buffer of runtime type from a pointer and some sizes.
const halide_buffer_t * raw_buffer() const
HALIDE_ALWAYS_INLINE Buffer< T2, D2, InClassDimStorage > & as() &
Return a typed reference to this Buffer.
HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept -> SliceOp< decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))>
ConstantInterval min(const ConstantInterval &a, const ConstantInterval &b)
ConstantInterval max(const ConstantInterval &a, const ConstantInterval &b)
bool any_zero(const Container &c)
BufferDeviceOwnership
This indicates how to deallocate the device for a Halide::Runtime::Buffer.
@ AllocatedDeviceAndHost
No free routine will be called when device ref count goes to zero
@ WrappedNative
halide_device_free will be called when device ref count goes to zero
@ Unmanaged
halide_device_detach_native will be called when device ref count goes to zero
@ Cropped
Call device_and_host_free when DevRefCount goes to zero.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr min(const FuncRef &a, const FuncRef &b)
Explicit overloads of min and max for FuncRef.
Expr max(const FuncRef &a, const FuncRef &b)
unsigned __INT64_TYPE__ uint64_t
__UINTPTR_TYPE__ uintptr_t
ALWAYS_INLINE T align_up(T p, size_t alignment)
unsigned __INT8_TYPE__ uint8_t
void * memset(void *s, int val, size_t n)
__PTRDIFF_TYPE__ ptrdiff_t
unsigned __INT16_TYPE__ uint16_t
unsigned __INT32_TYPE__ uint32_t
void * memcpy(void *s1, const void *s2, size_t n)
An iterator class, so that you can iterate over coordinates in a dimensions using a range-based for l...
bool operator!=(const iterator &other) const
A similar struct for managing device allocations.
BufferDeviceOwnership ownership
static void *(* default_allocate_fn)(size_t)
static void(* default_deallocate_fn)(void *)
The raw representation of an image passed around by generated Halide code.
int32_t dimensions
The dimensionality of the buffer.
halide_dimension_t * dim
The shape of the buffer.
uint64_t device
A device-handle for e.g.
uint8_t * host
A pointer to the start of the data in main memory.
struct halide_type_t type
The type of each buffer element.
const struct halide_device_interface_t * device_interface
The interface used to interpret the above handle.
Each GPU API provides a halide_device_interface_t struct pointing to the code that manages device all...
int(* device_slice)(void *user_context, const struct halide_buffer_t *src, int slice_dim, int slice_pos, struct halide_buffer_t *dst)
int(* device_and_host_malloc)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
int(* wrap_native)(void *user_context, struct halide_buffer_t *buf, uint64_t handle, const struct halide_device_interface_t *device_interface)
int(* device_release_crop)(void *user_context, struct halide_buffer_t *buf)
int(* device_crop)(void *user_context, const struct halide_buffer_t *src, struct halide_buffer_t *dst)
int(* copy_to_host)(void *user_context, struct halide_buffer_t *buf)
int(* copy_to_device)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
int(* device_free)(void *user_context, struct halide_buffer_t *buf)
int(* detach_native)(void *user_context, struct halide_buffer_t *buf)
int(* device_and_host_free)(void *user_context, struct halide_buffer_t *buf)
int(* device_malloc)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
A runtime tag for a type in the halide type system.