fix integer overflow in PIZ wavelet buffer arithmetic (#2328)

Three classes of signed integer overflow in the PIZ codec path, all
reachable from corrupt `dataWindow` dimensions in the EXR file header.

**`wav_2D_encode` / `wav_2D_decode` — wavelet loop pointer arithmetic**

`oy` is passed as `int` (value `wcount * nx`, at most ~INT32_MAX after
the guard below). Inside the hierarchical wavelet loop the expressions

    ey  = in + oy * (ny - p2)   // pointer end-of-row sentinel
    oy1 = oy * p                // row stride at level p
    oy2 = oy * p2               // row stride at level p2

multiply two values that can each approach INT32_MAX, producing a
signed 32-bit product that wraps to a small or negative value. The
wrapped value is used as a pointer offset, causing reads and writes
through `px` / `py` to land outside the allocated wavelet buffer.

Fix: widen by introducing `int64_t oy64 = oy` and using it for all
three expressions; `oy1` and `oy2` are also declared `int64_t`.

**`wavbuf += nx * ny * wcount` — per-channel buffer advance**

`nx`, `ny`, and `wcount` are all `int`. Their triple product overflows
int32 for moderately large images, causing subsequent channels to be
processed at an incorrect (too-small) offset into the wavelet buffer,
corrupting both encode and decode output.

Fix: cast to `(uint64_t)` before multiplying.

**`wcount * nx` — call-site argument overflow**

The fifth argument to `wav_2D_encode` / `wav_2D_decode` is `wcount * nx`
(`oy` = y-stride = elements per row). `wcount` is 1 or 2
(`bytes_per_element / 2`); for `wcount = 2` the product overflows int32
when `nx > INT32_MAX / 2`.

Fix: add an early bounds check `if (wcount > 0 && nx > INT_MAX / wcount)`
that rejects such input as `EXR_ERR_CORRUPT_CHUNK` before any arithmetic
is performed. This also keeps `wcount * nx` within int32 range at the
call site, ensuring `oy` arrives in the wavelet functions with a valid
non-overflowed value.

Made-with: Cursor

Signed-off-by: Cary Phillips <cary@ilm.com>

Author: cary-ilm

src/lib/OpenEXRCore/internal_piz.c

@@ -10,6 +10,7 @@
 #include "internal_huf.h"
 #include "internal_xdr.h"
 
+#include <limits.h>
 #include <string.h>
 
 /**************************************/
@@ -232,10 +233,11 @@ wdec16 (uint16_t l, uint16_t h, uint16_t* a, uint16_t* b)
 static void
 wav_2D_encode (uint16_t* in, int nx, int ox, int ny, int oy, uint16_t mx)
 {
-    int w14 = (mx < (1 << 14)) ? 1 : 0;
-    int n   = (nx > ny) ? ny : nx;
-    int p   = 1; // == 1 <<  level
-    int p2  = 2; // == 1 << (level+1)
+    int     w14  = (mx < (1 << 14)) ? 1 : 0;
+    int     n    = (nx > ny) ? ny : nx;
+    int     p    = 1; // == 1 <<  level
+    int     p2   = 2; // == 1 << (level+1)
+    int64_t oy64 = oy;
 
     //
     // Hierarchical loop on smaller dimension n
@@ -244,9 +246,9 @@ wav_2D_encode (uint16_t* in, int nx, int ox, int ny, int oy, uint16_t mx)
     while (p2 <= n)
     {
         uint16_t* py  = in;
-        uint16_t* ey  = in + oy * (ny - p2);
-        int       oy1 = oy * p;
-        int       oy2 = oy * p2;
+        uint16_t* ey  = in + oy64 * (ny - p2);
+        int64_t   oy1 = oy64 * p;
+        int64_t   oy2 = oy64 * p2;
         int       ox1 = ox * p;
         int       ox2 = ox * p2;
         uint16_t  i00, i01, i10, i11;
@@ -345,10 +347,11 @@ wav_2D_decode (
     int       oy, // i : y offset
     uint16_t  mx)  // i : maximum in[x][y] value
 {
-    int w14 = (mx < (1 << 14)) ? 1 : 0;
-    int n   = (nx > ny) ? ny : nx;
-    int p   = 1;
-    int p2;
+    int     w14  = (mx < (1 << 14)) ? 1 : 0;
+    int     n    = (nx > ny) ? ny : nx;
+    int     p    = 1;
+    int     p2;
+    int64_t oy64 = oy;
 
     //
     // Search max level
@@ -368,9 +371,9 @@ wav_2D_decode (
     while (p >= 1)
     {
         uint16_t* py  = in;
-        uint16_t* ey  = in + oy * (ny - p2);
-        int       oy1 = oy * p;
-        int       oy2 = oy * p2;
+        uint16_t* ey  = in + oy64 * (ny - p2);
+        int64_t   oy1 = oy64 * p;
+        int64_t   oy2 = oy64 * p2;
         int       ox1 = ox * p;
         int       ox2 = ox * p2;
         uint16_t  i00, i01, i10, i11;
@@ -566,11 +569,13 @@ internal_exr_apply_piz (exr_encode_pipeline_t* encode)
         nx     = curc->width;
         ny     = curc->height;
         wcount = (int) (curc->bytes_per_element / 2);
+        if (wcount > 0 && nx > INT_MAX / wcount)
+            return EXR_ERR_CORRUPT_CHUNK;
         for (int j = 0; j < wcount; ++j)
         {
             wav_2D_encode (wavbuf + j, nx, wcount, ny, wcount * nx, maxValue);
         }
-        wavbuf += nx * ny * wcount;
+        wavbuf += (uint64_t) nx * ny * wcount;
     }
 
     nBytes    = 0;
@@ -722,11 +727,13 @@ internal_exr_undo_piz (
         nx     = curc->width;
         ny     = curc->height;
         wcount = (int) (curc->bytes_per_element / 2);
+        if (wcount > 0 && nx > INT_MAX / wcount)
+            return EXR_ERR_CORRUPT_CHUNK;
         for (int j = 0; j < wcount; ++j)
         {
             wav_2D_decode (wavbuf + j, nx, wcount, ny, wcount * nx, maxValue);
         }
-        wavbuf += nx * ny * wcount;
+        wavbuf += (uint64_t) nx * ny * wcount;
     }
 
     //