Merge remote-tracking branch 'upstream/release_24.2' into dev

ahmedhamidawan · ahmedhamidawan · commit b80f41816ca8 · 2025-03-31T10:35:43.000-05:00
diff --git a/client/src/components/Form/Elements/FormData/FormData.vue b/client/src/components/Form/Elements/FormData/FormData.vue
@@ -22,6 +22,7 @@ import { type EventData, useEventStore } from "@/stores/eventStore";
 import { orList } from "@/utils/strings";
 
 import type { DataOption } from "./types";
+import { containsDataOption } from "./types";
 import { BATCH, SOURCE, VARIANTS } from "./variants";
 
 import FormSelection from "../FormSelection.vue";
@@ -384,49 +385,24 @@ function handleIncoming(incoming: Record<string, unknown> | Record<string, unkno
             const incomingValues: Array<DataOption> = [];
             values.forEach((currVal) => {
                 // Map incoming objects to data option values
-                const { newSrc, datasetCollectionDataset } = getSrcAndContentType(currVal);
-                let v: HistoryOrCollectionItem | HDAObject;
-                if (datasetCollectionDataset) {
-                    v = datasetCollectionDataset;
-                } else {
-                    v = currVal;
-                }
-                const newHid = isHistoryItem(v) ? v.hid : undefined;
-                const newId = v.id;
-                const newName = isHistoryItem(v) && v.name ? v.name : newId;
-                const newValue: DataOption = {
-                    id: newId,
-                    src: newSrc,
-                    batch: false,
-                    map_over_type: undefined,
-                    hid: newHid,
-                    name: newName,
-                    keep: true,
-                    tags: [],
-                };
-                if (isHistoryItem(v) && isHDCA(v) && props.collectionTypes?.length > 0) {
-                    const itemCollectionType = v.collection_type;
-                    if (!props.collectionTypes.includes(itemCollectionType as CollectionType)) {
-                        const mapOverType = props.collectionTypes.find((collectionType) =>
-                            itemCollectionType.endsWith(collectionType)
-                        );
-                        if (!mapOverType) {
-                            return false;
-                        }
-                        newValue["batch"] = true;
-                        newValue["map_over_type"] = mapOverType;
-                    }
+                const newValue = toDataOption(currVal);
+                if (!newValue) {
+                    return false;
                 }
                 // Verify that new value has corresponding option
-                const keepKey = `${newId}_${newSrc}`;
-                const existingOptions = props.options && props.options[newSrc];
-                const foundOption = existingOptions && existingOptions.find((option) => option.id === newId);
+                const keepKey = `${newValue.id}_${newValue.src}`;
+                const existingOptions = props.options && props.options[newValue.src];
+                const foundOption = existingOptions && existingOptions.find((option) => option.id === newValue.id);
                 if (!foundOption && !(keepKey in keepOptions)) {
-                    keepOptions[keepKey] = { label: `${newHid || "Selected"}: ${newName}`, value: newValue };
+                    keepOptions[keepKey] = {
+                        label: `${newValue.hid || "Selected"}: ${newValue.name}`,
+                        value: newValue,
+                    };
                 }
                 // Add new value to list
                 incomingValues.push(newValue);
             });
+            let hasDuplicates = false;
             if (incomingValues.length > 0 && incomingValues[0]) {
                 // Set new value
                 const config = currentVariant.value;
@@ -435,21 +411,68 @@ function handleIncoming(incoming: Record<string, unknown> | Record<string, unkno
                     if (config.multiple) {
                         const newValues = currentValue.value ? currentValue.value.slice() : [];
                         incomingValues.forEach((v) => {
-                            newValues.push(v);
+                            if (containsDataOption(newValues, v)) {
+                                hasDuplicates = true;
+                            } else {
+                                newValues.push(v);
+                            }
                         });
                         currentValue.value = newValues;
                     } else {
+                        if (containsDataOption(currentValue.value ?? [], firstValue)) {
+                            hasDuplicates = true;
+                        }
                         currentValue.value = [firstValue];
                     }
                 } else {
                     currentValue.value = incomingValues;
                 }
             }
+            if (hasDuplicates) {
+                return false;
+            }
         }
     }
     return true;
 }
 
+function toDataOption(item: HistoryOrCollectionItem): DataOption | null {
+    const { newSrc, datasetCollectionDataset } = getSrcAndContentType(item);
+    let v: HistoryOrCollectionItem | HDAObject;
+    if (datasetCollectionDataset) {
+        v = datasetCollectionDataset;
+    } else {
+        v = item;
+    }
+    const newHid = isHistoryItem(v) ? v.hid : undefined;
+    const newId = v.id;
+    const newName = isHistoryItem(v) && v.name ? v.name : newId;
+    const newValue: DataOption = {
+        id: newId,
+        src: newSrc,
+        batch: false,
+        map_over_type: undefined,
+        hid: newHid,
+        name: newName,
+        keep: true,
+        tags: [],
+    };
+    if (isHistoryItem(v) && isHDCA(v) && props.collectionTypes?.length > 0) {
+        const itemCollectionType = v.collection_type;
+        if (!props.collectionTypes.includes(itemCollectionType as CollectionType)) {
+            const mapOverType = props.collectionTypes.find((collectionType) =>
+                itemCollectionType.endsWith(collectionType)
+            );
+            if (!mapOverType) {
+                return null;
+            }
+            newValue["batch"] = true;
+            newValue["map_over_type"] = mapOverType;
+        }
+    }
+    return newValue;
+}
+
 /**
  * Open file dialog
  */
@@ -585,6 +608,16 @@ function isHistoryOrCollectionItem(item: EventData): item is HistoryOrCollection
     return isHistoryItem(item) || isDCE(item);
 }
 
+function getNameForItem(item: HistoryOrCollectionItem): string {
+    if (isHistoryItem(item)) {
+        return item.name ?? `Item ${item.hid}`;
+    } else if (isDCE(item)) {
+        return item.element_identifier;
+    } else {
+        throw new Error("Unknown item type");
+    }
+}
+
 // Drag/Drop event handlers
 function onDragEnter(evt: DragEvent) {
     const eventData = eventStore.getDragItems();
@@ -605,6 +638,13 @@ function onDragEnter(evt: DragEvent) {
                     highlightingState = "warning";
                     $emit("alert", `${historyContentType} is not an acceptable input type for this parameter.`);
                 }
+                // Check if the item is already in the current value
+                const option = toDataOption(item);
+                const isAlreadyInValue = containsDataOption(currentValue.value ?? [], option);
+                if (isAlreadyInValue) {
+                    highlightingState = "warning";
+                    $emit("alert", `${getNameForItem(item)} is already selected.`);
+                }
             }
         }
         currentHighlighting.value = highlightingState;
diff --git a/client/src/components/Form/Elements/FormData/types.ts b/client/src/components/Form/Elements/FormData/types.ts
@@ -13,3 +13,11 @@ export type DataOption = {
 export function isDataOption(item: object): item is DataOption {
     return !!item && "src" in item;
 }
+
+export function itemUniqueKey(item: DataOption): string {
+    return `${item.src}-${item.id}`;
+}
+
+export function containsDataOption(items: DataOption[], item: DataOption | null): boolean {
+    return item !== null && items.some((i) => itemUniqueKey(i) === itemUniqueKey(item));
+}
diff --git a/client/src/components/Form/Elements/FormSelect.vue b/client/src/components/Form/Elements/FormSelect.vue
@@ -9,7 +9,7 @@ import { useFilterObjectArray } from "@/composables/filter";
 import { useMultiselect } from "@/composables/useMultiselect";
 import { uid } from "@/utils/utils";
 
-import { type DataOption, isDataOption } from "./FormData/types";
+import { type DataOption, isDataOption, itemUniqueKey } from "./FormData/types";
 
 import StatelessTags from "@/components/TagsMultiselect/StatelessTags.vue";
 
@@ -154,10 +154,6 @@ const currentValue = computed({
     },
 });
 
-function itemUniqueKey(item: DataOption): string {
-    return `${item.src}-${item.id}`;
-}
-
 /**
  * Ensures that an initial value is selected for non-optional inputs
  */
diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
@@ -930,11 +930,12 @@ def set_meta(
             index_file = dataset.metadata.spec[spec_key].param.new_file(
                 dataset=dataset, metadata_tmp_files_dir=metadata_tmp_files_dir
             )
+        extra_threads = int(os.environ.get("GALAXY_SLOTS", 1)) - 1
         if index_flag == "-b":
             # IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4)
-            pysam.index("-o", index_file.get_file_name(), dataset.get_file_name())
+            pysam.index("-o", index_file.get_file_name(), f"-@{extra_threads}", dataset.get_file_name())  # type: ignore[attr-defined, unused-ignore]
         else:
-            pysam.index(index_flag, "-o", index_file.get_file_name(), dataset.get_file_name())
+            pysam.index(index_flag, "-o", index_file.get_file_name(), f"-@{extra_threads}", dataset.get_file_name())  # type: ignore[attr-defined, unused-ignore]
         dataset.metadata.bam_index = index_file
 
     def sniff(self, filename: str) -> bool:
@@ -1124,8 +1125,9 @@ def get_cram_version(self, filename: str) -> Tuple[int, int]:
             return -1, -1
 
     def set_index_file(self, dataset: HasFileName, index_file) -> bool:
+        extra_threads = int(os.environ.get("GALAXY_SLOTS", 1)) - 1
         try:
-            pysam.index("-o", index_file.get_file_name(), dataset.get_file_name())
+            pysam.index("-o", index_file.get_file_name(), f"-@{extra_threads}", dataset.get_file_name())  # type: ignore[attr-defined, unused-ignore]
             return True
         except Exception as exc:
             log.warning("%s, set_index_file Exception: %s", self, exc)
diff --git a/lib/galaxy/files/sources/invenio.py b/lib/galaxy/files/sources/invenio.py
@@ -403,12 +403,10 @@ def _get_download_file_url(self, record_id: str, filename: str, user_context: Op
         if is_draft_record:
             file_details_url = self._to_draft_url(file_details_url)
             download_file_content_url = self._to_draft_url(download_file_content_url)
-        file_details = self._get_response(user_context, file_details_url)
-        if not self._can_download_from_api(file_details):
-            # TODO: This is a temporary workaround for the fact that the "content" API
-            # does not support downloading files from S3 or other remote storage classes.
-            # More info: https://inveniordm.docs.cern.ch/reference/file_storage/#remote-files-r
-            download_file_content_url = f"{file_details_url.replace('/api', '')}?download=1"
+        # Downloading through the API is only supported for local files and depends on how
+        # the InvenioRDM instance file storage is configured.
+        # So this is the most reliable way to download files for now it.
+        download_file_content_url = f"{file_details_url.replace('/api', '')}?download=1"
         return download_file_content_url
 
     def _is_api_url(self, url: str) -> bool:
@@ -417,11 +415,6 @@ def _is_api_url(self, url: str) -> bool:
     def _to_draft_url(self, url: str) -> str:
         return url.replace("/files/", "/draft/files/")
 
-    def _can_download_from_api(self, file_details: dict) -> bool:
-        # Only files stored locally seems to be fully supported by the API for now
-        # More info: https://inveniordm.docs.cern.ch/reference/file_storage/
-        return file_details["storage_class"] == "L"
-
     def _is_draft_record(self, record_id: str, user_context: OptionalUserContext = None):
         request_url = self._get_draft_record_url(record_id)
         headers = self._get_request_headers(user_context)
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
@@ -229,9 +229,12 @@
 from galaxy.util.sanitize_html import sanitize_html
 
 if TYPE_CHECKING:
+    from sqlalchemy.sql.expression import BindParameter
+
     from galaxy.objectstore import (
         BaseObjectStore,
         ObjectStorePopulator,
+        QuotaSourceMap,
     )
     from galaxy.schema.invocation import InvocationMessageUnion
 
@@ -678,9 +681,9 @@ def stderr(self, stderr):
 """
 
 
-def calculate_user_disk_usage_statements(user_id, quota_source_map, for_sqlite=False):
+def calculate_user_disk_usage_statements(user_id: int, quota_source_map: "QuotaSourceMap", for_sqlite: bool = False):
     """Standalone function so can be reused for postgres directly in pgcleanup.py."""
-    statements = []
+    statements: List[Tuple[str, Dict[str, Any]]] = []
     default_quota_enabled = quota_source_map.default_quota_enabled
     default_exclude_ids = quota_source_map.default_usage_excluded_ids()
     default_cond = "dataset.object_store_id IS NULL" if default_quota_enabled and default_exclude_ids else ""
@@ -696,7 +699,7 @@ def calculate_user_disk_usage_statements(user_id, quota_source_map, for_sqlite=F
 UPDATE galaxy_user SET disk_usage = ({default_usage})
 WHERE id = :id
 """
-    params = {"id": user_id}
+    params: Dict[str, Any] = {"id": user_id}
     if default_exclude_ids:
         params["exclude_object_store_ids"] = default_exclude_ids
     statements.append((default_usage, params))
@@ -1162,25 +1165,27 @@ def calculate_disk_usage_default_source(self, object_store):
         usage = sa_session.scalar(sql_calc, params)
         return usage
 
-    def calculate_and_set_disk_usage(self, object_store):
+    def calculate_and_set_disk_usage(self, object_store: "BaseObjectStore"):
         """
         Calculates and sets user disk usage.
         """
         self._calculate_or_set_disk_usage(object_store=object_store)
 
-    def _calculate_or_set_disk_usage(self, object_store):
+    def _calculate_or_set_disk_usage(self, object_store: "BaseObjectStore"):
         """
         Utility to calculate and return the disk usage.  If dryrun is False,
         the new value is set immediately.
         """
         assert object_store is not None
         quota_source_map = object_store.get_quota_source_map()
         sa_session = object_session(self)
+        assert sa_session
+        assert sa_session.bind
         for_sqlite = "sqlite" in sa_session.bind.dialect.name
         statements = calculate_user_disk_usage_statements(self.id, quota_source_map, for_sqlite)
         for sql, args in statements:
             statement = text(sql)
-            binds = []
+            binds: List[BindParameter] = []
             for key, _ in args.items():
                 expand_binding = key.endswith("s")
                 binds.append(bindparam(key, expanding=expand_binding))
diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py
@@ -403,7 +403,7 @@ def to_dict(self) -> Dict[str, Any]:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def get_quota_source_map(self):
+    def get_quota_source_map(self) -> "QuotaSourceMap":
         """Return QuotaSourceMap describing mapping of object store IDs to quota sources."""
 
     @abc.abstractmethod
@@ -715,7 +715,7 @@ def parse_badges_from_config_xml(clazz, badges_xml):
             badges.append({"type": type, "message": message})
         return badges
 
-    def get_quota_source_map(self):
+    def get_quota_source_map(self) -> "QuotaSourceMap":
         # I'd rather keep this abstract... but register_singleton wants it to be instantiable...
         raise NotImplementedError()
 
diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py
@@ -8,6 +8,8 @@
     TYPE_CHECKING,
 )
 
+from packaging.version import Version
+
 from galaxy.tool_util.lint import Linter
 from galaxy.tool_util.parameters import validate_test_cases_for_tool_source
 from galaxy.tool_util.verify.assertion_models import assertion_list
@@ -166,10 +168,13 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
 class TestsCaseValidation(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+        profile = tool_source.parse_profile()
+        lint_log = lint_ctx.warn if Version(profile) < Version("24.2") else lint_ctx.error
+
         try:
             validation_results = validate_test_cases_for_tool_source(tool_source, use_latest_profile=True)
         except Exception as e:
-            lint_ctx.warn(
+            lint_log(
                 f"Serious problem parsing tool source or tests - cannot validate test cases. The exception is [{e}]",
                 linter=cls.name(),
             )
@@ -178,7 +183,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
             error = validation_result.validation_error
             if error:
                 error_str = _cleanup_pydantic_error(error)
-                lint_ctx.warn(
+                lint_log(
                     f"Test {test_idx}: failed to validate test parameters against inputs - tests won't run on a modern Galaxy tool profile version. Validation errors are [{error_str}]",
                     linter=cls.name(),
                 )
diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py
diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py
diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py
