From 83c6cdfa63ad61accec35575f503b629f94dc4af Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Thu, 19 Mar 2026 14:22:40 -0700 Subject: [PATCH] Fix B44/B44A integer overflow: use uint64_t for row offset (#2312) The B44 and B44A decoder and encoder use channel width (`nx`) and height (`ny`) in row pointer math. `nx` and `ny` are `int`; the scratch buffer is correctly sized with `(uint64_t)ny * (uint64_t)nx * bytes_per_element`, but row bases were computed as: ``` row0 = (uint16_t*)scratch; row0 += y * nx; // int * int -> signed overflow when y*nx > INT_MAX ``` For large `nx` (e.g. 268435456), `y*nx` overflows, so `row0`/`row1`/`row2`/`row3` point before the scratch buffer. Fix: compute the row offset in `uint64_t` before pointer arithmetic in both `uncompress_b44_impl` (decoder) and `compress_b44_impl` (encoder). Analysis and solution with the help of Curor / Claude Opus 4.5 Signed-off-by: Cary Phillips --- .../OpenEXR/OpenEXRCore/internal_b44.c | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c b/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c index 8d0c257e658..93279ff2ed5 100644 --- a/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c +++ b/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c @@ -390,13 +390,13 @@ compress_b44_impl (exr_encode_pipeline_t* encode, int flat_field) // rightmost column and the bottom row. // uint16_t *row0, *row1, *row2, *row3; + /* row offset in elements: use uint64_t so y*nx cannot overflow int */ + uint64_t row_off = (uint64_t) (y) * (uint64_t) (nx); - row0 = (uint16_t*) scratch; - row0 += y * nx; - - row1 = row0 + nx; - row2 = row1 + nx; - row3 = row2 + nx; + row0 = (uint16_t*) scratch + row_off; + row1 = row0 + (uint64_t) nx; + row2 = row1 + (uint64_t) nx; + row3 = row2 + (uint64_t) nx; if (y + 3 >= ny) { @@ -512,11 +512,12 @@ uncompress_b44_impl ( for (int y = 0; y < ny; y += 4) { - row0 = (uint16_t*) scratch; - row0 += y * nx; - row1 = row0 + nx; - row2 = row1 + nx; - row3 = row2 + nx; + /* row offset in elements: use uint64_t so y*nx cannot overflow int */ + uint64_t row_off = (uint64_t) (y) * (uint64_t) (nx); + row0 = (uint16_t*) scratch + row_off; + row1 = row0 + (uint64_t) nx; + row2 = row1 + (uint64_t) nx; + row3 = row2 + (uint64_t) nx; for (int x = 0; x < nx; x += 4) { if (bIn + 3 > comp_buf_size) return EXR_ERR_OUT_OF_MEMORY;