Opening HCS Plates from a Remote Store

iohub/cli/cli.py

@@ -6,6 +6,7 @@
 from iohub._version import __version__
 from iohub.cli.parsing import input_position_dirpaths
 from iohub.convert import TIFFConverter
+from iohub.ngff.nodes import _is_remote_url
 from iohub.reader import print_info
 from iohub.rename_wells import rename_wells
 
@@ -16,6 +17,24 @@
 )
 
 
+class _PathOrURL(click.ParamType):
+    """Accepts local directory paths or remote URLs."""
+
+    name = "PATH_OR_URL"
+
+    def convert(self, value, param, ctx):
+        if value is None:
+            return None
+        if _is_remote_url(value):
+            return value
+        path = pathlib.Path(value).resolve()
+        if not path.exists():
+            self.fail(f"'{value}' does not exist.", param, ctx)
+        if not path.is_dir():
+            self.fail(f"'{value}' is not a directory.", param, ctx)
+        return path
+
+
 @click.group()
 @click.help_option("-h", "--help")
 @click.version_option(version=VERSION)
@@ -29,7 +48,7 @@ def cli():
     "files",
     nargs=-1,
     required=True,
-    type=_DATASET_PATH,
+    type=_PathOrURL(),
 )
 @click.option(
     "--verbose",
@@ -39,14 +58,13 @@ def cli():
     "and full tree for HCS Plates in OME-Zarr",
 )
 def info(files, verbose):
-    """View basic metadata of a list of FILES.
+    """View basic metadata of a list of FILES or URLs.
 
-    Supported formats are Micro-Manager-acquired TIFF datasets
-    (single-page TIFF, multi-page OME-TIFF, NDTIFF)
-    and OME-Zarr (v0.1 linear HCS layout and all v0.4 layouts).
+    Supported formats: Micro-Manager TIFFs, OME-Zarr (v0.4/v0.5),
+    and remote URLs (HTTP/HTTPS, S3, GCS).
     """
     for file in files:
-        click.echo(f"Reading file:\t {file}")
+        click.echo(f"Reading:\t {file}")
         print_info(file, verbose=verbose)
 
 

iohub/ngff/nodes.py

@@ -18,6 +18,8 @@
 import zarr.codecs
 from numpy.typing import ArrayLike, DTypeLike, NDArray
 from pydantic import ValidationError
+from rich.console import Console
+from rich.tree import Tree as RichTree
 from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams
 from zarr.storage._utils import normalize_path
 
@@ -60,24 +62,46 @@ def _pad_shape(shape: tuple[int, ...], target: int = 5):
     return (1,) * pad + shape
 
 
+def _is_remote_url(path: str | Path) -> bool:
+    """Check if path is a remote URL (http, https, s3, gs, az)."""
+    url_str = str(path)
+    return url_str.startswith(
+        ('http://', 'https://', 's3://', 'gs://', 'az://')
+    )
+
+
 def _open_store(
     store_path: str | Path,
     mode: Literal["r", "r+", "a", "w", "w-"],
     version: Literal["0.4", "0.5"],
 ):
-    store_path = Path(store_path).resolve()
-    if not store_path.exists() and mode in ("r", "r+"):
-        raise FileNotFoundError(
-            f"Dataset directory not found at {str(store_path)}."
-        )
+    is_remote = _is_remote_url(store_path)
+
+    if is_remote:
+        if mode not in ("r",):
+            raise NotImplementedError(
+                f"Write mode '{mode}' not supported for remote URLs. "
+                "Use mode='r' for read-only access."
+            )
+    else:
+        # Local path: existing behavior
+        store_path = Path(store_path).resolve()
+        if not store_path.exists() and mode in ("r", "r+"):
+            raise FileNotFoundError(
+                f"Dataset directory not found at {str(store_path)}."
+            )
+
     if version not in ("0.4", "0.5"):
         _logger.warning(
             "IOHub is only tested against OME-NGFF v0.4 and v0.5. "
             f"Requested version {version} may not work properly."
         )
     try:
         zarr_format = None
-        if mode in ("w", "w-") or (mode == "a" and not store_path.exists()):
+        if not is_remote and (
+            mode in ("w", "w-")
+            or (mode == "a" and not Path(store_path).exists())
+        ):
             zarr_format = 3 if version == "0.5" else 2
         root = zarr.open_group(store_path, mode=mode, zarr_format=zarr_format)
     except Exception as e:
@@ -190,11 +214,13 @@ def __len__(self):
     def __getitem__(self, key):
         key = normalize_path(str(key))
         znode = self.zgroup.get(key)
-        if not znode:
+        if znode is None:
             raise KeyError(key)
         levels = len(key.split("/")) - 1
         item_type = self._MEMBER_TYPE
         for _ in range(levels):
+            if issubclass(item_type, ImageArray):
+                break
             item_type = item_type._MEMBER_TYPE
         if issubclass(item_type, ImageArray):
             return item_type.from_zarr_array(znode)
@@ -272,15 +298,34 @@ def is_leaf(self):
         """
         return not self.group_keys()
 
-    def print_tree(self, level: int | None = None):
-        """Print hierarchy of the node to stdout.
+    def _build_tree(
+        self,
+        parent: RichTree | None = None,
+        level: int | None = None,
+        current_level: int = 0,
+    ) -> RichTree:
+        """Build tree using OME-NGFF metadata (works for remote stores)."""
+        name = self.zgroup.basename or self.zgroup.name
+        tree = parent.add(name) if parent else RichTree(name)
+
+        if level is not None and current_level >= level:
+            return tree
+
+        for member_name in self._member_names:
+            member = self[member_name]
+            if isinstance(member, ImageArray):
+                shape = ", ".join(
+                    f"{a.name}: {s}" for a, s in zip(self.axes, member.shape)
+                )
+                tree.add(f"{member_name} ({shape})")
+            else:
+                member._build_tree(tree, level, current_level + 1)
 
-        Parameters
-        ----------
-        level : int, optional
-            Maximum depth to show, by default None
-        """
-        print(self.zgroup.tree(level=level))
+        return tree
+
+    def print_tree(self, level: int | None = None):
+        """Print hierarchy using OME-NGFF metadata"""
+        Console().print(self._build_tree(level=level))
 
     def iteritems(self):
         for key in self._member_names:
@@ -593,6 +638,7 @@ def __init__(
         axes: list[AxisMeta] | None = None,
         version: Literal["0.4", "0.5"] = "0.4",
         overwriting_creation: bool = False,
+        **kwargs,  # Absorbs extra attrs from parent hierarchy
     ):
         super().__init__(
             group=group,
@@ -640,7 +686,8 @@ def _zarr_format(self):
 
     @property
     def _member_names(self):
-        return self.array_keys()
+        """Array names from position metadata."""
+        return sorted(ds.path for ds in self.metadata.multiscales[0].datasets)
 
     @property
     def data(self):
@@ -1429,6 +1476,7 @@ def __init__(
         axes: list[AxisMeta] | None = None,
         version: Literal["0.4", "0.5"] = "0.4",
         overwriting_creation: bool = False,
+        **kwargs,  # Absorbs extra attrs from parent hierarchy
     ):
         super().__init__(
             group=group,
@@ -1467,6 +1515,11 @@ def __getitem__(self, key: str):
         """
         return super().__getitem__(key)
 
+    @property
+    def _member_names(self):
+        """Position names from well metadata."""
+        return sorted(img.path for img in self.metadata.images)
+
     def _create_position_nosync(self, name: str, acquisition: int = 0):
         "create_position, but doesn't write the metadata yet."
         pos_grp = self._group.create_group(name, overwrite=self._overwrite)
@@ -1541,6 +1594,7 @@ def __init__(
         axes: list[AxisMeta] | None = None,
         version: Literal["0.4", "0.5"] = "0.4",
         overwriting_creation: bool = False,
+        _plate_wells: list[WellIndexMeta] | None = None,
     ):
         super().__init__(
             group=group,
@@ -1550,6 +1604,7 @@ def __init__(
             version=version,
             overwriting_creation=overwriting_creation,
         )
+        self._plate_wells = _plate_wells
 
     def __getitem__(self, key: str):
         """Get a well member of the row.
@@ -1566,6 +1621,18 @@ def __getitem__(self, key: str):
         """
         return super().__getitem__(key)
 
+    @property
+    def _member_names(self):
+        """Column names from plate metadata."""
+        row_name = self.zgroup.basename
+        return sorted(
+            {
+                well.path.split("/")[1]
+                for well in self._plate_wells
+                if well.path.startswith(f"{row_name}/")
+            }
+        )
+
     def wells(self):
         """Returns a generator that iterate over the name and value
         of all the wells in the row.
@@ -1581,6 +1648,13 @@ def _parse_meta(self):
         # this node does not have NGFF metadata
         return
 
+    @property
+    def _child_attrs(self):
+        """Attributes to pass to Well children (exclude _plate_wells)."""
+        attrs = super()._child_attrs.copy()
+        attrs.pop("_plate_wells", None)
+        return attrs
+
 
 class Plate(NGFFNode):
     _MEMBER_TYPE = Row
@@ -1707,9 +1781,10 @@ def _first_pos_attr(self, attr: str):
         name = " ".join(attr.split("_")).strip()
         msg = f"Cannot determine {name}:"
         try:
-            row_grp = next(self.zgroup.groups())[1]
-            well_grp = next(row_grp.groups())[1]
-            pos_grp = next(well_grp.groups())[1]
+            pos_grp = self._get_first_position_group()
+            if pos_grp is None:
+                _logger.warning(f"{msg} No position is found in the dataset.")
+                return
         except StopIteration:
             _logger.warning(f"{msg} No position is found in the dataset.")
             return
@@ -1719,6 +1794,39 @@ def _first_pos_attr(self, attr: str):
         except AttributeError:
             _logger.warning(f"{msg} Invalid metadata at the first position")
 
+    def _get_first_position_group(self):
+        """Get the first position group using metadata."""
+        if not self.metadata or not self.metadata.wells:
+            raise ValueError(
+                "Plate metadata required to determine first position"
+            )
+
+        well_path = self.metadata.wells[0].path
+        well = Well(self.zgroup[well_path], parse_meta=True)
+
+        if not well.metadata or not well.metadata.images:
+            raise ValueError(
+                "Well metadata required to determine first position"
+            )
+
+        return self.zgroup[f"{well_path}/{well.metadata.images[0].path}"]
+
+    @property
+    def _member_names(self):
+        """Row names from plate metadata."""
+        return sorted(row.name for row in self.metadata.rows)
+
+    @property
+    def _child_attrs(self):
+        """Attributes to pass on when constructing child type instances."""
+        attrs = super()._child_attrs
+        # Pass only wells list - Row doesn't need full plate metadata
+        try:
+            attrs["_plate_wells"] = self.metadata.wells
+        except AttributeError:
+            pass
+        return attrs
+
     def dump_meta(self, field_count: bool = False):
         """Dumps metadata JSON to the `.zattrs` file.
 
@@ -1805,8 +1913,10 @@ def create_well(
         # normalize input
         row_name = normalize_path(row_name)
         col_name = normalize_path(col_name)
-        if row_name in self:
-            if col_name in self[row_name]:
+        # Check filesystem directly (not via _member_names/metadata)
+        # to ensure consistency during creation when metadata may be stale
+        if row_name in self.zgroup:
+            if col_name in self.zgroup[row_name]:
                 raise FileExistsError(
                     f"Well '{row_name}/{col_name}' already exists."
                 )
@@ -1824,15 +1934,15 @@ def create_well(
             self._build_meta(row_meta, col_meta, well_index_meta)
         else:
             self.metadata.wells.append(well_index_meta)
-        # create new row if needed
-        if row_name not in self:
+        # create new row if needed (check filesystem directly)
+        if row_name not in self.zgroup:
             row_grp = self.zgroup.create_group(
                 row_meta.name, overwrite=self._overwrite
             )
             if row_meta not in self.metadata.rows:
                 self.metadata.rows.append(row_meta)
         else:
-            row_grp = self[row_name].zgroup
+            row_grp = self.zgroup[row_name]
         if col_meta not in self.metadata.columns:
             self.metadata.columns.append(col_meta)
         # create well
@@ -2100,10 +2210,21 @@ def rename_well(self, old: str, new: str):
 
 
 def _check_file_mode(
-    store_path: Path,
+    store_path: str | Path,
     mode: Literal["r", "r+", "a", "w", "w-"],
     disable_path_checking: bool,
 ) -> bool:
+    if _is_remote_url(store_path):
+        # Remote URLs: only read mode supported, always parse metadata
+        if mode not in ("r",):
+            raise NotImplementedError(
+                f"Write mode '{mode}' not supported for remote URLs. "
+                "Use mode='r' for read-only access."
+            )
+        return True  # parse_meta = True for read mode
+
+    # Local paths: existing behavior
+    store_path = Path(store_path)
     if mode == "a":
         mode = "r+" if store_path.exists() else "w-"
     parse_meta = False
@@ -2265,7 +2386,9 @@ def open_ome_zarr(
         :py:class:`iohub.ngff.Plate`,
         or :py:class:`iohub.ngff.TiledPosition`)
     """
-    store_path = Path(store_path)
+    # Don't convert to Path for remote URLs
+    if not _is_remote_url(store_path):
+        store_path = Path(store_path)
     parse_meta = _check_file_mode(
         store_path, mode, disable_path_checking=disable_path_checking
     )