Merge main into feature/run-id-resume-main

Resolves conflicts in trackio/__init__.py and trackio/run.py, integrating
the Space-backed resume lookups from #506 with the run_id-based resume
logic on this branch. Space-backed init now queries /get_runs_for_project
and /get_run_summary via the RemoteClient and passes the resolved run_id
(when available) so step continuity works for multi-run-per-name projects.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: abidlabs

.changeset/few-bars-find.md

@@ -0,0 +1,5 @@
+---
+"trackio": patch
+---
+
+feat:Scope bucket sync to trackio/ subtree to avoid walking the HF cache

tests/e2e-spaces/test_sync_and_freeze.py

@@ -56,6 +56,10 @@ def _namespace_scoped_repo_id(test_space_id: str, repo_name: str) -> str:
     return repo_name
 
 
+def _repo_safe_suffix(nbytes: int = 6) -> str:
+    return secrets.token_hex(nbytes)
+
+
 def test_sync_to_gradio_space(test_space_id, temp_dir):
     project_name = f"test_sync_gradio_{secrets.token_urlsafe(8)}"
     run_name = "run1"
@@ -90,7 +94,7 @@ def test_sync_to_gradio_space(test_space_id, temp_dir):
 def test_sync_to_static_space_incremental(test_space_id, temp_dir):
     project_name = f"test_sync_static_{secrets.token_urlsafe(8)}"
     run_name = "run1"
-    suffix = secrets.token_urlsafe(6)
+    suffix = _repo_safe_suffix()
     space_id = _namespace_scoped_repo_id(test_space_id, f"trackio-test-static-{suffix}")
     space_id, _, bucket_id = utils.preprocess_space_and_dataset_ids(space_id, None)
 
@@ -137,7 +141,7 @@ def test_sync_gradio_then_freeze_to_static(test_space_id, temp_dir):
     client.predict(api_name="/force_sync")
     time.sleep(5)
 
-    suffix = secrets.token_urlsafe(6)
+    suffix = _repo_safe_suffix()
     frozen_space_id = _namespace_scoped_repo_id(
         test_space_id, f"trackio-test-frozen-{suffix}"
     )

trackio/__init__.py

@@ -97,7 +97,29 @@ def _cleanup_current_run():
             pass
 
 
-def _safe_get_runs_for_init(project: str) -> list[str]:
+def _safe_get_runs_for_init(
+    project: str,
+    space_id: str | None,
+    resume: str,
+    remote_client: RemoteClient | None = None,
+    check_existing_for_never: bool = False,
+) -> list[str]:
+    if space_id is not None:
+        if resume == "never" and not check_existing_for_never:
+            return []
+        try:
+            client = remote_client or RemoteClient(
+                space_id,
+                hf_token=huggingface_hub.utils.get_token(),
+                verbose=False,
+            )
+            runs = client.predict(project=project, api_name="/get_runs_for_project")
+            return runs if isinstance(runs, list) else []
+        except Exception as e:
+            _emit_nonfatal_warning(
+                f"trackio.init() could not inspect existing runs for project '{project}' on Space '{space_id}': {e}. Continuing without resume metadata."
+            )
+            return []
     try:
         return SQLiteStorage.get_runs(project)
     except Exception as e:
@@ -107,7 +129,32 @@ def _safe_get_runs_for_init(project: str) -> list[str]:
         return []
 
 
-def _safe_get_latest_run_for_init(project: str, name: str) -> dict | None:
+def _safe_get_latest_run_for_init(
+    project: str,
+    name: str,
+    space_id: str | None = None,
+    remote_client: RemoteClient | None = None,
+) -> dict | None:
+    if space_id is not None:
+        try:
+            client = remote_client or RemoteClient(
+                space_id,
+                hf_token=huggingface_hub.utils.get_token(),
+                verbose=False,
+            )
+            runs = client.predict(project=project, api_name="/get_runs_for_project")
+            if not isinstance(runs, list):
+                return None
+            matches = [r for r in runs if isinstance(r, dict) and r.get("name") == name]
+            if not matches:
+                return None
+            matches.sort(key=lambda r: r.get("created_at") or "", reverse=True)
+            return matches[0]
+        except Exception as e:
+            _emit_nonfatal_warning(
+                f"trackio.init() could not inspect existing runs for project '{project}' on Space '{space_id}': {e}. Continuing without resume metadata."
+            )
+            return None
     try:
         return SQLiteStorage.get_latest_run_record_by_name(project, name)
     except Exception as e:
@@ -117,6 +164,50 @@ def _safe_get_latest_run_for_init(project: str, name: str) -> dict | None:
         return None
 
 
+def _safe_get_last_step_for_init(
+    project: str,
+    run_name: str,
+    space_id: str | None,
+    resumed: bool,
+    run_id: str | None = None,
+    remote_client: RemoteClient | None = None,
+) -> int | None:
+    if not resumed:
+        return None
+    if space_id is not None:
+        try:
+            client = remote_client or RemoteClient(
+                space_id,
+                hf_token=huggingface_hub.utils.get_token(),
+                verbose=False,
+            )
+            summary_kwargs: dict[str, Any] = {
+                "project": project,
+                "api_name": "/get_run_summary",
+            }
+            if run_id is not None:
+                summary_kwargs["run_id"] = run_id
+            else:
+                summary_kwargs["run"] = run_name
+            summary = client.predict(**summary_kwargs)
+            if isinstance(summary, dict):
+                last_step = summary.get("last_step")
+                return last_step if isinstance(last_step, int) else None
+            return None
+        except Exception as e:
+            _emit_nonfatal_warning(
+                f"trackio.init() could not recover the previous step for run '{run_name}' on Space '{space_id}': {e}. Continuing from step 0."
+            )
+            return None
+    try:
+        return SQLiteStorage.get_max_step_for_run(project, run_name, run_id=run_id)
+    except Exception as e:
+        _emit_nonfatal_warning(
+            f"trackio.init() could not recover the previous step for run '{run_name}': {e}. Continuing from step 0."
+        )
+        return None
+
+
 def init(
     project: str,
     name: str | None = None,
@@ -298,8 +389,36 @@ def init(
                 )
     context_vars.current_project.set(project)
 
+    remote_client = None
+    if space_id is not None:
+        try:
+            remote_client = RemoteClient(
+                space_id,
+                hf_token=huggingface_hub.utils.get_token(),
+                verbose=False,
+            )
+        except Exception as e:
+            _emit_nonfatal_warning(
+                f"trackio.init() could not create a Space client for '{space_id}': {e}. Continuing with local fallback metadata lookups."
+            )
+
+    existing_run_records = _safe_get_runs_for_init(
+        project,
+        space_id,
+        resume,
+        remote_client=remote_client,
+        check_existing_for_never=name is not None,
+    )
+    existing_runs = [
+        r["name"] if isinstance(r, dict) else r for r in existing_run_records
+    ]
+
     existing_run = (
-        _safe_get_latest_run_for_init(project, name) if name is not None else None
+        _safe_get_latest_run_for_init(
+            project, name, space_id=space_id, remote_client=remote_client
+        )
+        if name is not None
+        else None
     )
     resolved_run_id = None
 
@@ -319,6 +438,19 @@ def init(
     else:
         raise ValueError("resume must be one of: 'must', 'allow', or 'never'")
 
+    initial_last_step = (
+        _safe_get_last_step_for_init(
+            project,
+            name,
+            space_id,
+            resumed,
+            run_id=resolved_run_id,
+            remote_client=remote_client,
+        )
+        if name is not None
+        else None
+    )
+
     if auto_log_gpu is None:
         nvidia_available = gpu_available()
         apple_available = apple_gpu_available()
@@ -342,6 +474,8 @@ def init(
         group=group,
         config=config,
         space_id=space_id,
+        existing_runs=existing_runs,
+        initial_last_step=initial_last_step,
         auto_log_gpu=auto_log_gpu,
         gpu_log_interval=gpu_log_interval,
         webhook_url=webhook_url,

trackio/bucket_storage.py

@@ -25,8 +25,8 @@ def _list_bucket_file_paths(bucket_id: str, prefix: str | None = None) -> list[s
 def download_bucket_to_trackio_dir(bucket_id: str) -> None:
     TRACKIO_DIR.mkdir(parents=True, exist_ok=True)
     sync_bucket(
-        source=f"hf://buckets/{bucket_id}",
-        dest=str(TRACKIO_DIR.parent),
+        source=f"hf://buckets/{bucket_id}/trackio",
+        dest=str(TRACKIO_DIR),
         quiet=True,
     )
 

trackio/run.py

@@ -43,6 +43,8 @@ def __init__(
         group: str | None = None,
         config: dict | None = None,
         space_id: str | None = None,
+        existing_runs: list[str] | None = None,
+        initial_last_step: int | None = None,
         auto_log_gpu: bool = False,
         gpu_log_interval: float = 10.0,
         webhook_url: str | None = None,
@@ -66,6 +68,9 @@ def __init__(
                 Keys starting with '_' are reserved for internal use.
             space_id: The HF Space ID if logging to a Space (e.g., "user/space").
                 If provided, media files will be uploaded to the Space.
+            existing_runs: Optional pre-fetched run names for this project. Used to
+                avoid redundant storage or remote lookups during init.
+            initial_last_step: Optional pre-fetched last step for a resumed run.
             auto_log_gpu: Whether to automatically log GPU metrics (utilization,
                 memory, temperature) at regular intervals.
             gpu_log_interval: The interval in seconds between GPU metric logs.
@@ -88,6 +93,8 @@ def __init__(
         self._client = client
         self._space_id = space_id
         self.id = run_id or uuid.uuid4().hex
+        self._existing_runs = existing_runs
+        self._initial_last_step = initial_last_step
         if name is not None:
             self.name = name
         else:
@@ -182,6 +189,8 @@ def _warn_once(self, key: str, message: str) -> None:
         _emit_nonfatal_warning(message)
 
     def _safe_get_existing_runs(self) -> list[str]:
+        if self._existing_runs is not None:
+            return self._existing_runs
         try:
             return SQLiteStorage.get_runs(self.project)
         except Exception as e:
@@ -192,6 +201,8 @@ def _safe_get_existing_runs(self) -> list[str]:
             return []
 
     def _safe_get_max_step_for_run(self) -> int | None:
+        if self._initial_last_step is not None:
+            return self._initial_last_step
         try:
             return SQLiteStorage.get_max_step_for_run(
                 self.project, self.name, run_id=self.id