Merge remote-tracking branch 'upstream/release_25.0' into merge_25.0_into_dev_sept

Author: ahmedhamidawan

.github/workflows/integration.yaml

@@ -64,7 +64,7 @@ jobs:
         uses: medyagh/setup-minikube@latest
         with:
           driver: none
-          kubernetes-version: '1.23.0'
+          kubernetes-version: '1.28.0'
       - name: Check pods
         run: kubectl get pods -A
       - uses: actions/checkout@v4

client/src/components/History/Content/GenericItem.vue

@@ -1,5 +1,11 @@
 <template>
-    <component :is="providerComponent" :id="itemId" v-slot="{ result: item, loading }" auto-refresh>
+    <component
+        :is="providerComponent"
+        :id="itemId"
+        :key="view"
+        v-slot="{ result: item, loading }"
+        :view="view"
+        auto-refresh>
         <LoadingSpan v-if="loading" message="Loading dataset" />
         <div v-else>
             <ContentItem
@@ -11,7 +17,7 @@
                 :expand-dataset="expandDataset"
                 :is-dataset="item.history_content_type == 'dataset' || item.element_type == 'hda'"
                 @update:expand-dataset="expandDataset = $event"
-                @view-collection="viewCollection = !viewCollection"
+                @view-collection="onViewCollection"
                 @delete="onDelete"
                 @toggleHighlights="onHighlight(item)"
                 @undelete="onUndelete(item)"
@@ -60,6 +66,7 @@ export default {
         return {
             viewCollection: false,
             expandDataset: false,
+            view: this.itemSrc === "hdca" ? "collection" : "element",
         };
     },
     computed: {
@@ -124,6 +131,12 @@ export default {
                 this.onError(error, "Failed to highlight related items");
             }
         },
+        onViewCollection(collection) {
+            if (this.view === "collection" && collection.model_class === "HistoryDatasetCollectionAssociation") {
+                this.view = "element";
+            }
+            this.viewCollection = !this.viewCollection;
+        },
     },
 };
 </script>

client/src/components/Tool/ToolForm.vue

@@ -87,7 +87,7 @@
                     id="execute"
                     class="text-nowrap"
                     title="Run Tool"
-                    :disabled="!canMutateHistory"
+                    :disabled="runButtonDisabled"
                     size="small"
                     :wait="showExecuting"
                     :tooltip="tooltip"
@@ -97,7 +97,7 @@
                 <ButtonSpinner
                     title="Run Tool"
                     class="mt-3 mb-3"
-                    :disabled="!canMutateHistory"
+                    :disabled="runButtonDisabled"
                     :wait="showExecuting"
                     :tooltip="tooltip"
                     @onClick="onExecute(config, currentHistoryId)" />
@@ -205,6 +205,7 @@ export default {
             ],
             immutableHistoryMessage:
                 "This history is immutable and you cannot run tools in it. Please switch to a different history.",
+            formConfigInitialized: false,
         };
     },
     computed: {
@@ -227,6 +228,12 @@ export default {
             if (!this.canMutateHistory) {
                 return this.immutableHistoryMessage;
             }
+            if (this.hasConfigOrValErrors) {
+                return "Please resolve highlighted issues before running the tool.";
+            }
+            if (this.showExecuting) {
+                return "Tool is being executed...";
+            }
             return `Run tool: ${this.formConfig.name} (${this.formConfig.version})`;
         },
         errorContentPretty() {
@@ -252,8 +259,15 @@ export default {
         canMutateHistory() {
             return this.currentHistory && canMutateHistory(this.currentHistory);
         },
-        runButtonTitle() {
-            return "Run Tool";
+        runButtonDisabled() {
+            return this.disabled || !this.canMutateHistory || this.hasConfigOrValErrors;
+        },
+        /** If there are any backend returned `formConfig.errors` or internal/client checked validation errors. */
+        hasConfigOrValErrors() {
+            return (
+                (this.formConfig.errors && Object.values(this.formConfig.errors).length > 0) ||
+                this.validationInternal?.length
+            );
         },
     },
     watch: {
@@ -286,7 +300,12 @@ export default {
             this.formData = newData;
             if (refreshRequest) {
                 this.onUpdate();
+            } else if (this.formConfigInitialized && this.hasConfigOrValErrors) {
+                // After the first manual change to a form input, for every change, if there isn't a request to refresh,
+                // we reset the errors since we haven't received a tool form update via the backend.
+                this.formConfig.errors = null;
             }
+            this.formConfigInitialized = true;
         },
         onUpdate() {
             this.disabled = true;

client/src/components/providers/DatasetCollectionProvider.js

@@ -1,10 +1,15 @@
 import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
 
-import { fetchCollectionDetails } from "@/api/datasetCollections";
+import { fetchCollectionDetails, fetchCollectionSummary } from "@/api/datasetCollections";
 
 // There isn't really a good way to know when to stop polling for HDCA updates,
 // but we know the populated_state should at least be ok.
 export default SingleQueryProvider(
-    (params) => fetchCollectionDetails({ hdca_id: params.id }),
+    (params) => {
+        if (params.view && params.view === "collection") {
+            return fetchCollectionSummary({ hdca_id: params.id });
+        }
+        return fetchCollectionDetails({ hdca_id: params.id });
+    },
     (result) => result.populated_state === "ok",
 );

client/src/composables/useInvocationGraph.ts

@@ -10,7 +10,7 @@ import {
 } from "@fortawesome/free-solid-svg-icons";
 import { computed, type Ref, ref, set } from "vue";
 
-import { fetchCollectionDetails } from "@/api/datasetCollections";
+import { fetchCollectionSummary } from "@/api/datasetCollections";
 import { fetchDatasetDetails } from "@/api/datasets";
 import type { InvocationStep, StepJobSummary, WorkflowInvocationElementView } from "@/api/invocations";
 import type { StoredWorkflowDetailed } from "@/api/workflows";
@@ -323,7 +323,7 @@ export function useInvocationGraph(
                 set(graphStep, "state", getContentItemState(hda));
                 set(graphStep, "nodeText", `${hda.hid}: <b>${hda.name}</b>`);
             } else {
-                const hdca = await fetchCollectionDetails({ hdca_id: inputItem.id });
+                const hdca = await fetchCollectionSummary({ hdca_id: inputItem.id });
                 // TODO: Same type mismatch as above
                 set(graphStep, "state", getContentItemState(hdca));
                 set(graphStep, "nodeText", `${hdca.hid}: <b>${hdca.name}</b>`);

doc/source/admin/production.md

@@ -194,5 +194,5 @@ The list of tasks that are currently handled by `Celery` can be found in `lib/ga
 To enable Celery in your instance you need to follow some additional steps:
 
 -   Set `enable_celery_tasks: true` in the Galaxy config.
--   Configure the `backend` under `celery_conf` to store the results of the tasks. For example, you can use [`redis` as the backend](https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#broker-redis). If you are using `redis`, make sure to install the `redis` dependency in your Galaxy environment with `pip install redis`. You can find more information on how to configure other backends in the [Celery documentation](https://docs.celeryq.dev/en/stable/userguide/tasks.html#task-result-backends).
+-   Configure the `backend` under `celery_conf` to store the results of the tasks. For example, you can use [`redis` as the backend](https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#broker-redis). If you are using `redis`, make sure to install the `redis` dependency in your Galaxy environment with `pip install redis`. You can find more information on how to configure other backends in the [Celery documentation](https://docs.celeryq.dev/en/stable/userguide/tasks.html#task-result-backends). Keep in mind that you should not reuse the main Galaxy database as a backend for Celery.
 -   Configure one or more workers to handle the tasks. You can find more information on how to configure workers in the [Celery documentation](https://docs.celeryq.dev/en/stable/userguide/workers.html). If you are using [Gravity](https://github.com/galaxyproject/gravity) it will simplify the process of setting up Celery workers.

lib/galaxy/authnz/custos_authnz.py

@@ -17,6 +17,7 @@
 import jwt
 from oauthlib.common import generate_nonce
 from requests_oauthlib import OAuth2Session
+from sqlalchemy import func
 
 from galaxy import (
     exceptions,
@@ -121,11 +122,19 @@ def refresh(self, trans, custos_authnz_token):
             return False
         if not custos_authnz_token.refresh_token:
             return False
-        refresh_token_decoded = self._decode_token_no_signature(custos_authnz_token.refresh_token)
-        # do not attempt to use refresh token that is already expired
-        if int(refresh_token_decoded["exp"]) <= int(time.time()):
-            # in the future we might want to log out the user here
+
+        # Try to extract expiration date from the refresh token. If expired, do not refresh token.
+        try:
+            refresh_token_decoded = self._decode_token_no_signature(custos_authnz_token.refresh_token)
+            # do not attempt to use refresh token that is already expired
+            if int(refresh_token_decoded["exp"]) <= int(time.time()):
+                # in the future we might want to log out the user here
+                return False
+        except jwt.exceptions.DecodeError:
+            log.warning("Refresh token cannot be decoded. Galaxy does not support non-decodable refresh tokens.")
+            # If the refresh token is non-decodable, we do not use it because we cannot reliably determine its expiration date. See discussion in https://github.com/galaxyproject/galaxy/pull/20821
             return False
+
         oauth2_session = self._create_oauth2_session()
         token_endpoint = self.config.token_endpoint
         if self.config.iam_client_secret:
@@ -237,7 +246,7 @@ def callback(self, state_token, authz_code, trans, login_redirect_url):
         custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config.provider)
         if custos_authnz_token is None:
             user = trans.user
-            existing_user = trans.sa_session.query(User).filter_by(email=email).first()
+            existing_user = trans.sa_session.query(User).where(func.lower(User.email) == email.lower()).first()
             if not user:
                 if existing_user:
                     if trans.app.config.fixed_delegated_auth:

lib/galaxy/config/sample/datatypes_conf.xml.sample

@@ -234,11 +234,12 @@
     <datatype extension="h5ad" type="galaxy.datatypes.binary:Anndata" description="An HDF5-based anndata File" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="h5mu" type="galaxy.datatypes.binary:H5" mimetype="application/octet-stream" display_in_upload="true" subclass="true" description="MuData is a format analogous to Anndata for multimodal datasets and is based on HDF5" description_url="https://github.com/scverse/mudata"/>
     <datatype extension="visium.tar.gz" type="galaxy.datatypes.binary:Visium" subclass="true" display_in_upload="true" description="Visium is a tar.gz archive with at least a 'Spatial' subfolder, a filtered h5 file and a raw h5 file." />
-    <datatype extension="mz5" type="galaxy.datatypes.binary:H5" subclass="true" mimetype="application/octet-stream" display_in_upload="true" description="Mz5 is an HDF5-based open mass spectrometry file format modeled after mzML." /> 
+    <datatype extension="mz5" type="galaxy.datatypes.binary:H5" subclass="true" mimetype="application/octet-stream" display_in_upload="true" description="Mz5 is an HDF5-based open mass spectrometry file format modeled after mzML." />
     <datatype extension="mzmlb" type="galaxy.datatypes.binary:H5" subclass="true" mimetype="application/octet-stream" display_in_upload="true" description="MzMLb is an HDF5-based open mass spectrometry file format that stores metadata as mzML and binary data as native HDF5 types." />
     <datatype extension="mbi" type="galaxy.datatypes.binary:H5" subclass="true" mimetype="application/octet-stream" display_in_upload="true" description="MBI is MOBILion's proprietary HDF5-based format for its ion mobility mass spectrometry data." />
     <datatype extension="hyphy_results.json" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
     <datatype extension="hivtrace" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
+    <datatype extension="hic" type="galaxy.datatypes.binary:Hic" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="cool" type="galaxy.datatypes.binary:Cool" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="mcool" type="galaxy.datatypes.binary:MCool" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="h5mlm" type="galaxy.datatypes.binary:H5MLM" mimetype="application/octet-stream" display_in_upload="true"/>
@@ -698,6 +699,9 @@
     <datatype extension="docx" type="galaxy.datatypes.binary:Docx" display_in_upload="true" decription="DOCX is an XML-based file format that is natively used for Microsoft Word documents" description_url="https://www.iso.org/standard/71691.html">
         <infer_from suffix="docx" />
     </datatype>
+    <datatype extension="pptx" type="galaxy.datatypes.binary:Pptx" display_in_upload="true" decription="PPTX is an XML-based file format that is natively used for Microsoft PowerPoint presentations" description_url="https://www.iso.org/standard/71691.html">
+        <infer_from suffix="pptx" />
+    </datatype>
     <datatype extension="btwisted" type="galaxy.datatypes.data:Text" subclass="true"/>
     <datatype extension="cai" type="galaxy.datatypes.data:Text" subclass="true"/>
     <datatype extension="cat_db" type="galaxy.datatypes.data:Text" subclass="true"/>
@@ -1228,6 +1232,7 @@
     <sniffer type="galaxy.datatypes.binary:NcbiTaxonomySQlite"/>
     <sniffer type="galaxy.datatypes.binary:SQlite"/>
     <sniffer type="galaxy.datatypes.binary:H5MLM"/>
+    <sniffer type="galaxy.datatypes.binary:Hic"/>
     <sniffer type="galaxy.datatypes.binary:Cool"/>
     <sniffer type="galaxy.datatypes.binary:MCool"/>
     <sniffer type="galaxy.datatypes.binary:Loom"/>
@@ -1273,6 +1278,8 @@
     <sniffer type="galaxy.datatypes.binary:Edr"/>
     <sniffer type="galaxy.datatypes.binary:Vel"/>
     <sniffer type="galaxy.datatypes.binary:Xlsx"/>
+    <sniffer type="galaxy.datatypes.binary:Docx"/>
+    <sniffer type="galaxy.datatypes.binary:Pptx"/>
     <sniffer type="galaxy.datatypes.binary:Numpy"/>
     <sniffer type="galaxy.datatypes.qiime2:QIIME2Metadata"/>
     <sniffer type="galaxy.datatypes.qiime2:QIIME2Artifact"/>

lib/galaxy/datatypes/binary.py

@@ -3192,6 +3192,7 @@ class Docx(Binary):
 
     file_ext = "docx"
     compressed = True
+    display_behavior = "download"  # Office documents trigger downloads
 
     def sniff_prefix(self, file_prefix: FilePrefix) -> bool:
         # Docx is compressed in zip format and must not be uncompressed in Galaxy.
@@ -3214,6 +3215,22 @@ def sniff_prefix(self, file_prefix: FilePrefix) -> bool:
         return file_prefix.compressed_mime_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 
 
+@build_sniff_from_prefix
+class Pptx(Binary):
+    """Class for PowerPoint 2007 (pptx) files"""
+
+    file_ext = "pptx"
+    compressed = True
+    display_behavior = "download"  # Office documents trigger downloads
+
+    def sniff_prefix(self, file_prefix: FilePrefix) -> bool:
+        # Pptx is compressed in zip format and must not be uncompressed in Galaxy.
+        return (
+            file_prefix.compressed_mime_type
+            == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+        )
+
+
 @build_sniff_from_prefix
 class ExcelXls(Binary):
     """Class describing an Excel (xls) file"""
@@ -4778,3 +4795,63 @@ def display_peek(self, dataset: DatasetProtocol) -> str:
             return dataset.peek
         except Exception:
             return f"Binary numpy file ({nice_size(dataset.get_size())})"
+
+
+@build_sniff_from_prefix
+class Hic(Binary):
+    """
+    Hic: highly compressed binary file that stores contact matrices
+    from multiple resolutions in a clever way, allowing random access.
+    https://github.com/aidenlab/hic-format
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('merlin.hic')
+    >>> Hic().sniff(fname)
+    True
+    >>> fname = get_test_fname('test.mz5')
+    >>> Hic().sniff(fname)
+    False
+    """
+
+    file_ext = "hic"
+
+    MetadataElement(
+        name="version",
+        default="",
+        param=MetadataParameter,
+        desc="Version of the HiC file format",
+        readonly=True,
+        visible=True,
+        no_value=0,
+        optional=True,
+    )
+
+    def __init__(self, **kwd):
+        super().__init__(**kwd)
+        self._magic = b"HIC"
+
+    def sniff_prefix(self, file_prefix: FilePrefix) -> bool:
+        return file_prefix.startswith_bytes(self._magic)
+
+    def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
+        if not dataset.dataset.purged:
+            dataset.peek = "Binary HiC file"
+            dataset.blurb = f"{nice_size(dataset.get_size())}"
+            dataset.blurb += f"\nHiC Format v{dataset.metadata.version}"
+        else:
+            dataset.peek = "file does not exist"
+            dataset.blurb = "file purged from disk"
+
+    def display_peek(self, dataset: DatasetProtocol) -> str:
+        try:
+            return dataset.peek
+        except Exception:
+            return f"Binary HiC file ({nice_size(dataset.get_size())})"
+
+    def set_meta(self, dataset: DatasetProtocol, overwrite: bool = True, **kwd) -> None:
+        """
+        Set metadata for HiC file.
+        """
+        with open(dataset.get_file_name(), "rb") as handle:
+            header_bytes = handle.read(8)
+        dataset.metadata.version = struct.unpack("<i", header_bytes[4:8])[0]

lib/galaxy/datatypes/sniff.py

@@ -51,7 +51,12 @@
 log = logging.getLogger(__name__)
 
 SNIFF_PREFIX_BYTES = int(os.environ.get("GALAXY_SNIFF_PREFIX_BYTES", None) or 2**20)
-BINARY_MIMETYPES = {"application/pdf", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}
+BINARY_MIMETYPES = {
+    "application/pdf",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+}
 
 
 def get_test_fname(fname):