Internal libtiff: add TIFFGetMaxCompressionRatio() and use it in _TIFFReadEncoded[Tile|Strip)AndAllocBuffer()

frmts/gtiff/libtiff/tif_compress.c

@@ -169,6 +169,18 @@ void _TIFFSetDefaultPostDecode(TIFF *tif)
     }
 }
 
+static uint64_t _TIFFDefaultGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    return 0; /* unknown */
+}
+
+static uint64_t _TIFFGetMaxCompressionRatioOne(TIFF *tif)
+{
+    (void)tif;
+    return 1; /* no compression */
+}
+
 void _TIFFSetDefaultCompressionState(TIFF *tif)
 {
     tif->tif_fixuptags = _TIFFNoFixupTags;
@@ -191,6 +203,7 @@ void _TIFFSetDefaultCompressionState(TIFF *tif)
     tif->tif_cleanup = _TIFFvoid;
     tif->tif_defstripsize = _TIFFDefaultStripSize;
     tif->tif_deftilesize = _TIFFDefaultTileSize;
+    tif->tif_getmaxcompressionratio = _TIFFDefaultGetMaxCompressionRatio;
     tif->tif_flags &= ~(TIFF_NOBITREV | TIFF_NOREADRAW);
 }
 
@@ -199,6 +212,8 @@ int TIFFSetCompressionScheme(TIFF *tif, int scheme)
     const TIFFCodec *c = TIFFFindCODEC((uint16_t)scheme);
 
     _TIFFSetDefaultCompressionState(tif);
+    if (scheme == COMPRESSION_NONE)
+        tif->tif_getmaxcompressionratio = _TIFFGetMaxCompressionRatioOne;
     /*
      * Don't treat an unknown compression scheme as an error.
      * This permits applications to open files with data that
@@ -208,6 +223,13 @@ int TIFFSetCompressionScheme(TIFF *tif, int scheme)
     return (c ? (*c->init)(tif, scheme) : 1);
 }
 
+uint64_t TIFFGetMaxCompressionRatio(TIFF *tif)
+{
+    if (tif->tif_getmaxcompressionratio)
+        return tif->tif_getmaxcompressionratio(tif);
+    return 0;
+}
+
 /*
  * Other compression schemes may be registered.  Registered
  * schemes can also override the builtin versions provided

frmts/gtiff/libtiff/tif_fax3.c

@@ -1518,6 +1518,18 @@ static void Fax3PrintDir(TIFF *tif, FILE *fd, long flags)
         (*sp->printdir)(tif, fd, flags);
 }
 
+static uint64_t Fax3GetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    /* 1024x1024: 36 */
+    /* 4096x4096: 100 */
+    /* 16383x16383: 163 */
+    /* 65536x65536: 200 */
+    /* 200000x200000: 208 */
+
+    return 250;
+}
+
 static int InitCCITTFax3(TIFF *tif)
 {
     static const char module[] = "InitCCITTFax3";
@@ -1582,6 +1594,7 @@ static int InitCCITTFax3(TIFF *tif)
     tif->tif_encodetile = Fax3Encode;
     tif->tif_close = Fax3Close;
     tif->tif_cleanup = Fax3Cleanup;
+    tif->tif_getmaxcompressionratio = Fax3GetMaxCompressionRatio;
 
     return (1);
 }
@@ -1731,6 +1744,12 @@ static int Fax4PostEncode(TIFF *tif)
     return (1);
 }
 
+static uint64_t Fax4GetMaxCompressionRatio(TIFF *tif)
+{
+    return isTiled(tif) ? tif->tif_dir.td_tilewidth
+                        : tif->tif_dir.td_imagewidth;
+}
+
 int TIFFInitCCITTFax4(TIFF *tif, int scheme)
 {
     (void)scheme;
@@ -1753,6 +1772,7 @@ int TIFFInitCCITTFax4(TIFF *tif, int scheme)
         tif->tif_encodestrip = Fax4Encode;
         tif->tif_encodetile = Fax4Encode;
         tif->tif_postencode = Fax4PostEncode;
+        tif->tif_getmaxcompressionratio = Fax4GetMaxCompressionRatio;
         /*
          * Suppress RTC at the end of each strip.
          */
@@ -1824,6 +1844,18 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s)
     return (1);
 }
 
+static uint64_t Fax3RLEGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    /* 1024x1024: 43 */
+    /* 4096x4096: 128 */
+    /* 16383x16383: 171 */
+    /* 65536x65536: 205 */
+    /* 200000x200000: 211 */
+
+    return 250;
+}
+
 int TIFFInitCCITTRLE(TIFF *tif, int scheme)
 {
     (void)scheme;
@@ -1832,6 +1864,7 @@ int TIFFInitCCITTRLE(TIFF *tif, int scheme)
         tif->tif_decoderow = Fax3DecodeRLE;
         tif->tif_decodestrip = Fax3DecodeRLE;
         tif->tif_decodetile = Fax3DecodeRLE;
+        tif->tif_getmaxcompressionratio = Fax3RLEGetMaxCompressionRatio;
         /*
          * Suppress RTC+EOLs when encoding and byte-align data.
          */
@@ -1850,6 +1883,7 @@ int TIFFInitCCITTRLEW(TIFF *tif, int scheme)
         tif->tif_decoderow = Fax3DecodeRLE;
         tif->tif_decodestrip = Fax3DecodeRLE;
         tif->tif_decodetile = Fax3DecodeRLE;
+        tif->tif_getmaxcompressionratio = Fax3RLEGetMaxCompressionRatio;
         /*
          * Suppress RTC+EOLs when encoding and word-align data.
          */

frmts/gtiff/libtiff/tif_jpeg.c

@@ -2809,6 +2809,30 @@ static int JPEGInitializeLibJPEG(TIFF *tif, int decompress)
     return 1;
 }
 
+static uint64_t JPEGGetMaxCompressionRatio(TIFF *tif)
+{
+    JPEGState *sp = JState(tif);
+    if ((tif->tif_dir.td_photometric == PHOTOMETRIC_YCBCR) &&
+        (tif->tif_dir.td_planarconfig == PLANARCONFIG_CONTIG) &&
+        (tif->tif_dir.td_samplesperpixel == 3))
+    {
+        if (sp->h_sampling == 2 && sp->v_sampling == 2)
+        {
+            if (tif->tif_dir.td_bitspersample == 12)
+                return 768;
+            else
+                return 512;
+        }
+
+        return 0; /* unknown */
+    }
+
+    if (tif->tif_dir.td_bitspersample == 12)
+        return 384;
+    else
+        return 256;
+}
+
 /* Common to tif_jpeg.c and tif_jpeg_12.c */
 static void TIFFInitJPEGCommon(TIFF *tif)
 {
@@ -2845,6 +2869,7 @@ static void TIFFInitJPEGCommon(TIFF *tif)
     tif->tif_encoderow = JPEGEncode;
     tif->tif_encodestrip = JPEGEncode;
     tif->tif_encodetile = JPEGEncode;
+    tif->tif_getmaxcompressionratio = JPEGGetMaxCompressionRatio;
     tif->tif_cleanup = JPEGCleanup;
 
     tif->tif_defstripsize = JPEGDefaultStripSize;

frmts/gtiff/libtiff/tif_lerc.c

@@ -1488,6 +1488,16 @@ static int LERCVGetField(TIFF *tif, uint32_t tag, va_list ap)
     return 1;
 }
 
+static uint64_t LERCGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+
+    /* LERC compression ratio can grow to several millions */
+    /* eg. 5703725 for Lerc deflate on 16383x16383 array */
+    /* or 3829644 for regular Lerc */
+    return 0;
+}
+
 int TIFFInitLERC(TIFF *tif, int scheme)
 {
     static const char module[] = "TIFFInitLERC";
@@ -1538,6 +1548,7 @@ int TIFFInitLERC(TIFF *tif, int scheme)
     tif->tif_encodestrip = LERCEncode;
     tif->tif_encodetile = LERCEncode;
 #endif
+    tif->tif_getmaxcompressionratio = LERCGetMaxCompressionRatio;
     tif->tif_cleanup = LERCCleanup;
 
     /* Default values for codec-specific fields */

frmts/gtiff/libtiff/tif_lzma.c

@@ -486,6 +486,17 @@ static const TIFFField lzmaFields[] = {
      FALSE, "LZMA2 Compression Preset", NULL},
 };
 
+static uint64_t LZMAGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    /* 1024x1024: 3800 */
+    /* 4096x4096: 6534 */
+    /* 16383x16383: 6846 */
+    /* 65536x65536: 6874 */
+
+    return 7000;
+}
+
 int TIFFInitLZMA(TIFF *tif, int scheme)
 {
     static const char module[] = "TIFFInitLZMA";
@@ -561,6 +572,8 @@ int TIFFInitLZMA(TIFF *tif, int scheme)
     tif->tif_encodestrip = LZMAEncode;
     tif->tif_encodetile = LZMAEncode;
     tif->tif_cleanup = LZMACleanup;
+    tif->tif_getmaxcompressionratio = LZMAGetMaxCompressionRatio;
+
     /*
      * Setup predictor setup.
      */

frmts/gtiff/libtiff/tif_lzw.c

@@ -1399,6 +1399,17 @@ static void LZWCleanup(TIFF *tif)
     _TIFFSetDefaultCompressionState(tif);
 }
 
+static uint64_t LZWGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    /* 1024x1024: 562 */
+    /* 4096x4096: 1243 */
+    /* 16383x16383: 1353 */
+    /* 65536x65536: 1362 */
+
+    return 1400;
+}
+
 int TIFFInitLZW(TIFF *tif, int scheme)
 {
     static const char module[] = "TIFFInitLZW";
@@ -1432,6 +1443,7 @@ int TIFFInitLZW(TIFF *tif, int scheme)
     tif->tif_encodestrip = LZWEncode;
     tif->tif_encodetile = LZWEncode;
 #endif
+    tif->tif_getmaxcompressionratio = LZWGetMaxCompressionRatio;
     tif->tif_cleanup = LZWCleanup;
     /*
      * Setup predictor setup.

frmts/gtiff/libtiff/tif_packbits.c

@@ -317,6 +317,12 @@ static int PackBitsDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s)
     return (1);
 }
 
+static uint64_t PackBitsGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    return 64;
+}
+
 int TIFFInitPackBits(TIFF *tif, int scheme)
 {
     (void)scheme;
@@ -330,6 +336,8 @@ int TIFFInitPackBits(TIFF *tif, int scheme)
     tif->tif_encodestrip = PackBitsEncodeChunk;
     tif->tif_encodetile = PackBitsEncodeChunk;
 #endif
+    tif->tif_getmaxcompressionratio = PackBitsGetMaxCompressionRatio;
+
     return (1);
 }
 #endif /* PACKBITS_SUPPORT */

frmts/gtiff/libtiff/tif_pixarlog.c

@@ -1619,6 +1619,15 @@ static const TIFFField pixarlogFields[] = {
     {TIFFTAG_PIXARLOGQUALITY, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, FIELD_PSEUDO,
      FALSE, FALSE, "", NULL}};
 
+static uint64_t PixarLogGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    /* cf https://zlib.net/zlib_tech.html */
+    const uint64_t MAX_DEFLATE_RATIO = 1032;
+
+    /* security margin as I don't understand what this codec does */
+    return MAX_DEFLATE_RATIO * (uint64_t)4;
+}
 int TIFFInitPixarLog(TIFF *tif, int scheme)
 {
     static const char module[] = "TIFFInitPixarLog";
@@ -1666,6 +1675,7 @@ int TIFFInitPixarLog(TIFF *tif, int scheme)
     tif->tif_encodetile = PixarLogEncode;
     tif->tif_close = PixarLogClose;
     tif->tif_cleanup = PixarLogCleanup;
+    tif->tif_getmaxcompressionratio = PixarLogGetMaxCompressionRatio;
 
     /* Override SetField so we can handle our private pseudo-tag */
     sp->vgetparent = tif->tif_tagmethods.vgetfield;

frmts/gtiff/libtiff/tif_read.c

@@ -618,13 +618,31 @@ tmsize_t _TIFFReadEncodedStripAndAllocBuffer(TIFF *tif, uint32_t strip,
     if (!TIFFFillStrip(tif, strip))
         return ((tmsize_t)(-1));
 
-    *buf = _TIFFmallocExt(tif, bufsizetoalloc);
+    /* Sanity checks to avoid excessive memory allocation */
+    /* Max compression ratio experimentally determined. Might be fragile...
+     * Only apply this heuristics to situations where the memory allocation
+     * would be big, to avoid breaking nominal use cases.
+     */
+    const uint64_t maxCompressionRatio = TIFFGetMaxCompressionRatio(tif);
+    if (maxCompressionRatio > 0 && bufsizetoalloc > 100 * 1000 * 1000 &&
+        (uint64_t)tif->tif_rawdatasize <
+            (uint64_t)this_stripsize / maxCompressionRatio)
+    {
+        TIFFErrorExtR(tif, TIFFFileName(tif),
+                      "Likely invalid strip byte count for strip %u. "
+                      "Uncompressed strip size is %" PRIu64 ", "
+                      "compressed one is %" PRIu64,
+                      strip, (uint64_t)this_stripsize,
+                      (uint64_t)tif->tif_rawdatasize);
+        return ((tmsize_t)(-1));
+    }
+
+    *buf = _TIFFcallocExt(tif, 1, bufsizetoalloc);
     if (*buf == NULL)
     {
         TIFFErrorExtR(tif, TIFFFileName(tif), "No space for strip buffer");
         return ((tmsize_t)(-1));
     }
-    _TIFFmemset(*buf, 0, bufsizetoalloc);
 
     if ((*tif->tif_decodestrip)(tif, (uint8_t *)*buf, this_stripsize, plane) <=
         0)
@@ -1088,17 +1106,10 @@ tmsize_t _TIFFReadEncodedTileAndAllocBuffer(TIFF *tif, uint32_t tile,
          * Only apply this heuristics to situations where the memory allocation
          * would be big, to avoid breaking nominal use cases.
          */
-        const int maxCompressionRatio =
-            td->td_compression == COMPRESSION_ZSTD ? 33000
-            : td->td_compression == COMPRESSION_JXL
-                ?
-                /* Evaluated on a 8000x8000 tile */
-                25000 * (td->td_planarconfig == PLANARCONFIG_CONTIG
-                             ? td->td_samplesperpixel
-                             : 1)
-                : td->td_compression == COMPRESSION_LZMA ? 7000 : 1000;
-        if (bufsizetoalloc > 100 * 1000 * 1000 &&
-            tif->tif_rawdatasize < tilesize / maxCompressionRatio)
+        const uint64_t maxCompressionRatio = TIFFGetMaxCompressionRatio(tif);
+        if (maxCompressionRatio > 0 && bufsizetoalloc > 100 * 1000 * 1000 &&
+            (uint64_t)tif->tif_rawdatasize <
+                (uint64_t)tilesize / maxCompressionRatio)
         {
             TIFFErrorExtR(tif, TIFFFileName(tif),
                           "Likely invalid tile byte count for tile %u. "

frmts/gtiff/libtiff/tif_webp.c

@@ -847,6 +847,14 @@ static const TIFFField TWebPFields[] = {
      FIELD_PSEUDO, TRUE, FALSE, "WEBP exact lossless", NULL},
 };
 
+static uint64_t TWebPGetMaxCompressionRatio(TIFF *tif)
+{
+    /* lossy compression: */
+    /* return (tif->tif_dir.td_samplesperpixel == 4) ? 2199 : 1685; */
+    /* lossless compression: */
+    return (tif->tif_dir.td_samplesperpixel == 4) ? 104194 : 78146;
+}
+
 int TIFFInitWebP(TIFF *tif, int scheme)
 {
     static const char module[] = "TIFFInitWebP";
@@ -910,6 +918,7 @@ int TIFFInitWebP(TIFF *tif, int scheme)
     tif->tif_encodestrip = TWebPEncode;
     tif->tif_encodetile = TWebPEncode;
     tif->tif_cleanup = TWebPCleanup;
+    tif->tif_getmaxcompressionratio = TWebPGetMaxCompressionRatio;
 
     return 1;
 bad:

frmts/gtiff/libtiff/tif_zip.c

@@ -700,6 +700,13 @@ static void TIFF_zfree(void *opaque, void *ptr)
     _TIFFfreeExt((TIFF *)opaque, ptr);
 }
 
+static uint64_t ZIPGetMaxCompressionRatio(TIFF *tif)
+{
+    (void)tif;
+    /* cf https://zlib.net/zlib_tech.html */
+    return 1032;
+}
+
 int TIFFInitZIP(TIFF *tif, int scheme)
 {
     static const char module[] = "TIFFInitZIP";
@@ -766,6 +773,7 @@ int TIFFInitZIP(TIFF *tif, int scheme)
     tif->tif_encodestrip = ZIPEncode;
     tif->tif_encodetile = ZIPEncode;
     tif->tif_cleanup = ZIPCleanup;
+    tif->tif_getmaxcompressionratio = ZIPGetMaxCompressionRatio;
     /*
      * Setup predictor setup.
      */