Add flag to selectively tolerate missing data during exports

Author: guerler

lib/galaxy/managers/model_stores.py

@@ -264,7 +264,11 @@ def write_history_to(self, request: WriteHistoryTo):
         uri: Optional[str] = None
         try:
             export_store = model.store.get_export_store_factory(
-                self._app, model_store_format, export_files=export_files, user_context=user_context
+                self._app,
+                model_store_format,
+                export_files=export_files,
+                user_context=user_context,
+                tolerate_missing_data=True,
             )(request.target_uri)
             with export_store:
                 history = self._history_manager.by_id(request.history_id)

lib/galaxy/model/__init__.py

@@ -459,6 +459,7 @@ def __init__(
         serialize_dataset_objects: Optional[bool] = None,
         serialize_files_handler: Optional[SerializeFilesHandler] = None,
         strip_metadata_files: Optional[bool] = None,
+        tolerate_missing_data: bool = False,
     ) -> None:
         self.for_edit = for_edit
         if serialize_dataset_objects is None:
@@ -470,6 +471,10 @@ def __init__(
             # expect metadata tool to be rerun.
             strip_metadata_files = not for_edit
         self.strip_metadata_files = strip_metadata_files
+        # When True, serializers emit best-effort output for histories whose imports left
+        # unresolved references (orphan ImplicitCollectionJobsJobAssociation rows, null-id
+        # job param refs) instead of raising. Intended for background archival exports.
+        self.tolerate_missing_data = tolerate_missing_data
 
     def attach_identifier(self, id_encoder, obj, ret_val):
         if self.for_edit and obj.id:
@@ -492,7 +497,11 @@ def get_identifier(self, id_encoder, obj):
             return obj.temp_id
 
     def get_identifier_for_id(self, id_encoder, obj_id):
-        if not obj_id or self.for_edit:
+        if not obj_id:
+            if self.tolerate_missing_data:
+                return obj_id
+            raise NotImplementedError()
+        if self.for_edit:
             return obj_id
         return id_encoder.encode_id(obj_id, kind="model_export")
 
@@ -2950,7 +2959,9 @@ def _serialize(self, id_encoder, serialization_options):
             self,
             populated_state=self.populated_state,
             jobs=[
-                serialization_options.get_identifier(id_encoder, j_a.job) for j_a in self.jobs if j_a.job is not None
+                serialization_options.get_identifier(id_encoder, j_a.job)
+                for j_a in self.jobs
+                if j_a.job is not None or not serialization_options.tolerate_missing_data
             ],
         )
         serialization_options.attach_identifier(id_encoder, self, rval)

lib/galaxy/model/store/__init__.py

@@ -1965,6 +1965,7 @@ def __init__(
         strip_metadata_files: bool = True,
         serialize_jobs: bool = True,
         user_context=None,
+        tolerate_missing_data: bool = False,
     ) -> None:
         """
         :param export_directory: path to export directory. Will be created if it does not exist.
@@ -2001,6 +2002,7 @@ def __init__(
             serialize_dataset_objects=serialize_dataset_objects,
             strip_metadata_files=strip_metadata_files,
             serialize_files_handler=self,
+            tolerate_missing_data=tolerate_missing_data,
         )
         self.export_files = export_files
         self.included_datasets: dict[model.DatasetInstance, tuple[model.DatasetInstance, bool]] = {}
@@ -3040,13 +3042,15 @@ def get_export_store_factory(
     export_files=None,
     bco_export_options: Optional[BcoExportOptions] = None,
     user_context=None,
+    tolerate_missing_data: bool = False,
 ) -> Callable[[StrPath], FileSourceModelExportStore]:
     export_store_class: type[FileSourceModelExportStore]
     export_store_class_kwds = {
         "app": app,
         "export_files": export_files,
         "serialize_dataset_objects": False,
         "user_context": user_context,
+        "tolerate_missing_data": tolerate_missing_data,
     }
     if download_format in ["tar.gz", "tgz"]:
         export_store_class = TarModelExportStore

test/unit/data/model/test_model.py

@@ -58,10 +58,10 @@ def test_io_dicts_excludes_implicit_output_collections():
     assert "paired_output" in io.out_collections
 
 
-def test_implicit_collection_jobs_serialize_skips_orphan_associations():
-    """ImplicitCollectionJobsJobAssociation.job_id is nullable; orphan rows
-    (e.g. produced by partial imports) must be skipped on export instead of
-    crashing in SerializationOptions.get_identifier on a None job."""
+def test_implicit_collection_jobs_serialize_orphan_associations():
+    """ImplicitCollectionJobsJobAssociation.job_id is nullable; orphan rows can
+    be persisted by the import path. Strict mode (default) lets the crash
+    propagate; tolerate_missing_data mode skips the orphan."""
     icj = model.ImplicitCollectionJobs()
     job = model.Job()
     job.id = 42
@@ -73,15 +73,31 @@ def test_implicit_collection_jobs_serialize_skips_orphan_associations():
     orphan.order_index = 1  # job left as None
     icj.jobs = [linked, orphan]
 
-    rval = icj._serialize(id_encoder=None, serialization_options=model.SerializationOptions(for_edit=True))
+    strict = model.SerializationOptions(for_edit=True)
+    try:
+        icj._serialize(id_encoder=None, serialization_options=strict)
+    except AttributeError:
+        pass
+    else:
+        raise AssertionError("strict mode should have raised on orphan ICJJA")
+
+    tolerant = model.SerializationOptions(for_edit=True, tolerate_missing_data=True)
+    rval = icj._serialize(id_encoder=None, serialization_options=tolerant)
     assert rval["jobs"] == [42]
 
 
-def test_job_serialize_passes_through_null_param_ids():
-    """Job params can contain {"src": "hda"|"hdca"|"dce", "id": null} references
-    (e.g. from corrupted state); export must pass null ids through instead of
-    raising NotImplementedError in get_identifier_for_id."""
-    options = model.SerializationOptions(for_edit=True)
-    assert options.get_identifier_for_id(id_encoder=None, obj_id=None) is None
-    assert options.get_identifier_for_id(id_encoder=None, obj_id=0) == 0
-    assert options.get_identifier_for_id(id_encoder=None, obj_id=42) == 42
+def test_get_identifier_for_id_null_handling():
+    """Null job-param ids raise in strict mode and pass through in
+    tolerate_missing_data mode."""
+    strict = model.SerializationOptions(for_edit=True)
+    try:
+        strict.get_identifier_for_id(id_encoder=None, obj_id=None)
+    except NotImplementedError:
+        pass
+    else:
+        raise AssertionError("strict mode should have raised on null obj_id")
+
+    tolerant = model.SerializationOptions(for_edit=True, tolerate_missing_data=True)
+    assert tolerant.get_identifier_for_id(id_encoder=None, obj_id=None) is None
+    assert tolerant.get_identifier_for_id(id_encoder=None, obj_id=0) == 0
+    assert tolerant.get_identifier_for_id(id_encoder=None, obj_id=42) == 42