Resolve Gallery base64 representation issues

CHANGELOG.md

@@ -16,7 +16,7 @@ No changes to highlight.
 No changes to highlight.
 
 ## Full Changelog:
-No changes to highlight.
+* Speeds up Gallery component by using temporary files instead of base64 representation in the front-end by [@proxyphi](https://github.com/proxyphi), [@pngwn](https://github.com/pngwn), and [@abidlabs](https://github.com/abidlabs) in [PR 2265](https://github.com/gradio-app/gradio/pull/2265)
 
 ## Contributors Shoutout:
 No changes to highlight.

gradio/components.py

@@ -3224,6 +3224,7 @@ def __init__(
             visible: If False, component will be hidden.
             elem_id: An optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.
         """
+        self.temp_dir = tempfile.mkdtemp()
         super().__init__(
             label=label,
             show_label=show_label,
@@ -3265,7 +3266,7 @@ def postprocess(
         Parameters:
             y: list of images, or list of (image, caption) tuples
         Returns:
-            list of base64 data or (base64 image data, caption) pairs
+            list of string file paths to images in temp directory
         """
         if y is None:
             return []
@@ -3275,19 +3276,26 @@ def postprocess(
             if isinstance(img, tuple) or isinstance(img, list):
                 img, caption = img
             if isinstance(img, np.ndarray):
-                img = processing_utils.encode_array_to_base64(img)
+                file = processing_utils.save_array_to_file(img, dir=self.temp_dir)
             elif isinstance(img, PIL.Image.Image):
-                img = processing_utils.encode_pil_to_base64(img)
+                file = processing_utils.save_pil_to_file(img, dir=self.temp_dir)
             elif isinstance(img, str):
-                img = processing_utils.encode_url_or_file_to_base64(img)
+                if utils.validate_url(img):
+                    file = processing_utils.download_to_file(img, dir=self.temp_dir)
+                else:
+                    file = processing_utils.create_tmp_copy_of_file(
+                        img, dir=self.temp_dir
+                    )
             else:
-                raise ValueError(
-                    "Unknown type. Please choose from: 'numpy', 'pil', 'file'."
-                )
+                raise ValueError(f"Cannot process type as image: {type(img)}")
+
             if caption is not None:
-                output.append([img, caption])
+                output.append(
+                    [{"name": file.name, "data": None, "is_file": True}, caption]
+                )
             else:
-                output.append(img)
+                output.append({"name": file.name, "data": None, "is_file": True})
+
         return output
 
     def style(
@@ -3322,15 +3330,11 @@ def deserialize(
         for img_data in x:
             if isinstance(img_data, list) or isinstance(img_data, tuple):
                 img_data, caption = img_data
-                prefix = f"[{utils.strip_invalid_filename_characters(caption)}]-"
             else:
                 caption = None
-                prefix = None
-            file_obj = processing_utils.decode_base64_to_file(
-                img_data, dir=gallery_path, encryption_key=encryption_key, prefix=prefix
-            )
+            name = FileSerializable.deserialize(self, img_data, gallery_path)
             if caption is not None:
-                captions[file_obj.name] = caption
+                captions[name] = caption
         if len(captions):
             captions_file = os.path.join(gallery_path, "captions.json")
             with open(captions_file, "w") as captions_json:
@@ -3350,7 +3354,7 @@ def serialize(self, x: Any, load_dir: str = "", called_directly: bool = False):
                 caption = captions.get(file_path)
             else:
                 caption = None
-            img = ImgSerializable.serialize(self, file_path)
+            img = FileSerializable.serialize(self, file_path)
             if caption:
                 files.append([img, caption])
             else:

gradio/external.py

@@ -20,6 +20,7 @@
 
 import gradio
 from gradio import components, exceptions, utils
+from gradio.processing_utils import to_binary
 
 if TYPE_CHECKING:
     from gradio.components import DataframeData
@@ -158,9 +159,7 @@ def encode_to_base64(r: requests.Response) -> str:
             # example model: ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition
             "inputs": components.Audio(source="upload", type="filepath", label="Input"),
             "outputs": components.Label(label="Class"),
-            "preprocess": lambda i: base64.b64decode(
-                i["data"].split(",")[1]
-            ),  # convert the base64 representation to binary
+            "preprocess": lambda i: to_binary,
             "postprocess": lambda r: postprocess_label(
                 {i["label"].split(", ")[0]: i["score"] for i in r.json()}
             ),
@@ -169,18 +168,14 @@ def encode_to_base64(r: requests.Response) -> str:
             # example model: speechbrain/mtl-mimic-voicebank
             "inputs": components.Audio(source="upload", type="filepath", label="Input"),
             "outputs": components.Audio(label="Output"),
-            "preprocess": lambda i: base64.b64decode(
-                i["data"].split(",")[1]
-            ),  # convert the base64 representation to binary
+            "preprocess": to_binary,
             "postprocess": encode_to_base64,
         },
         "automatic-speech-recognition": {
             # example model: jonatasgrosman/wav2vec2-large-xlsr-53-english
             "inputs": components.Audio(source="upload", type="filepath", label="Input"),
             "outputs": components.Textbox(label="Output"),
-            "preprocess": lambda i: base64.b64decode(
-                i["data"].split(",")[1]
-            ),  # convert the base64 representation to binary
+            "preprocess": to_binary,
             "postprocess": lambda r: r.json()["text"],
         },
         "feature-extraction": {
@@ -202,9 +197,7 @@ def encode_to_base64(r: requests.Response) -> str:
             # Example: google/vit-base-patch16-224
             "inputs": components.Image(type="filepath", label="Input Image"),
             "outputs": components.Label(label="Classification"),
-            "preprocess": lambda i: base64.b64decode(
-                i.split(",")[1]
-            ),  # convert the base64 representation to binary
+            "preprocess": to_binary,
             "postprocess": lambda r: postprocess_label(
                 {i["label"].split(", ")[0]: i["score"] for i in r.json()}
             ),

gradio/processing_utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import base64
 import json
 import mimetypes
@@ -8,6 +10,7 @@
 import tempfile
 import warnings
 from io import BytesIO
+from typing import Dict
 
 import numpy as np
 import requests
@@ -21,9 +24,23 @@
     from pydub import AudioSegment
 
 
+#########################
+# GENERAL
+#########################
+
+
+def to_binary(x: str | Dict) -> bytes:
+    """Converts a base64 string or dictionary to a binary string that can be sent in a POST."""
+    if isinstance(x, dict):
+        x = encode_url_or_file_to_base64(x["name"])
+    return base64.b64decode(x.split(",")[1])
+
+
 #########################
 # IMAGE PRE-PROCESSING
 #########################
+
+
 def decode_base64_to_image(encoding):
     content = encoding.split(";")[1]
     image_encoded = content.split(",")[1]