Merge pull request #21646 from bernt-matthias/ssh-fs-improvements

Improvements for ssh file sources

Author: bernt-matthias

client/src/api/fileSources.ts

@@ -87,6 +87,10 @@ export const templateTypes: FileSourceTypesDetail = {
         icon: faNetworkWired,
         message: "This is a file repository plugin that connects with an OMERO server.",
     },
+    ssh: {
+        icon: faNetworkWired,
+        message: "This is a file repository plugin that connects with a remote server over SSH.",
+    },
 };
 
 export const FileSourcesValidFilters = {

client/src/api/schema/schema.ts

@@ -12847,7 +12847,8 @@ export interface components {
                 | "dataverse"
                 | "huggingface"
                 | "iiif"
-                | "omero";
+                | "omero"
+                | "ssh";
             /** Variables */
             variables?:
                 | (
@@ -23022,6 +23023,8 @@ export interface components {
             help?: string | null;
             /** Label */
             label?: string | null;
+            /** Multiline */
+            multiline?: boolean | null;
             /** Name */
             name: string;
             /** Optional */
@@ -23035,6 +23038,8 @@ export interface components {
             help?: string | null;
             /** Label */
             label?: string | null;
+            /** Multiline */
+            multiline?: boolean | null;
             /** Name */
             name: string;
             /** Optional */
@@ -23061,6 +23066,8 @@ export interface components {
             help?: string | null;
             /** Label */
             label?: string | null;
+            /** Multiline */
+            multiline?: boolean | null;
             /** Name */
             name: string;
             /** Optional */
@@ -23087,6 +23094,8 @@ export interface components {
             help?: string | null;
             /** Label */
             label?: string | null;
+            /** Multiline */
+            multiline?: boolean | null;
             /** Name */
             name: string;
             /** Optional */
@@ -23113,6 +23122,8 @@ export interface components {
             help?: string | null;
             /** Label */
             label?: string | null;
+            /** Multiline */
+            multiline?: boolean | null;
             /** Name */
             name: string;
             /** Optional */
@@ -24487,7 +24498,8 @@ export interface components {
                 | "dataverse"
                 | "huggingface"
                 | "iiif"
-                | "omero";
+                | "omero"
+                | "ssh";
             /** Uri Root */
             uri_root: string;
             /**

client/src/components/ConfigTemplates/EditSecretsForm.vue

@@ -29,6 +29,7 @@ async function update(secretName: string, secretValue: string) {
                     :name="secret.name"
                     :help="secret.help || ''"
                     :is-set="true"
+                    :multiline="secret.multiline || false"
                     @update="update">
                 </VaultSecret>
             </div>

client/src/components/ConfigTemplates/VaultSecret.test.ts

@@ -23,4 +23,18 @@ describe("VaultSecret", () => {
         // verify markdown converted
         expect(helpWrapper.html()).toEqual("<p>here is some good <em>help</em></p>");
     });
+
+    it("should render a textarea editor for multiline secrets", async () => {
+        const wrapper = shallowMount(VaultSecret as object, {
+            propsData: {
+                name: "secret name",
+                label: "Label Secret",
+                help: "pem help",
+                isSet: true,
+                multiline: true,
+            },
+            localVue,
+        });
+        expect(wrapper.html()).toContain("bformtextarea-stub");
+    });
 });

client/src/components/ConfigTemplates/VaultSecret.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { faPen } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
-import { BButton, BFormInput, BInputGroup, BInputGroupAppend } from "bootstrap-vue";
+import { BButton, BFormInput, BFormTextarea, BInputGroup, BInputGroupAppend } from "bootstrap-vue";
 import { computed, ref } from "vue";
 
 import { markup } from "@/components/ObjectStore/configurationMarkdown";
@@ -11,6 +11,7 @@ interface Props {
     label: string;
     help: string;
     isSet: boolean;
+    multiline?: boolean;
 }
 const props = defineProps<Props>();
 
@@ -57,7 +58,8 @@ async function onOk() {
         </div>
         <b-modal ref="edit-modal" v-model="showEdit" :title="editTitle" ok-title="Update" @ok="onOk">
             <div>
-                <BFormInput v-model="secretValue" type="password" />
+                <BFormTextarea v-if="multiline" v-model="secretValue" rows="8" no-resize />
+                <BFormInput v-else v-model="secretValue" type="password" />
                 <!-- eslint-disable-next-line vue/no-v-html -->
                 <span class="ui-form-info form-text text-muted" v-html="helpHtml" />
             </div>

client/src/components/ConfigTemplates/formUtil.test.ts

@@ -122,6 +122,17 @@ describe("formUtils", () => {
             const formEntry = templateVariableFormEntry(requiredIntVar, undefined);
             expect(formEntry.value).toBe(0);
         });
+        it("should render multiline string types as Galaxy textarea inputs", () => {
+            const multilineVariable: TemplateVariable = {
+                name: "private_key",
+                type: "string",
+                multiline: true,
+            };
+            const formEntry = templateVariableFormEntry(multilineVariable, "line1\nline2");
+            expect(formEntry.type).toBe("text");
+            expect(formEntry.area).toBe(true);
+            expect(formEntry.value).toBe("line1\nline2");
+        });
     });
 
     describe("templateVariableFormEntry optional field", () => {
@@ -184,6 +195,17 @@ describe("formUtils", () => {
             const formEntry = templateSecretFormEntry(secretWithoutOptional);
             expect(formEntry.optional).toBe(false);
         });
+
+        it("should render multiline secrets as textarea inputs", () => {
+            const multilineSecret: TemplateSecret = {
+                name: "private_key",
+                help: "PEM content",
+                multiline: true,
+            };
+            const formEntry = templateSecretFormEntry(multilineSecret);
+            expect(formEntry.type).toBe("text");
+            expect(formEntry.area).toBe(true);
+        });
     });
 
     describe("formDataTypedGet", () => {

client/src/components/ConfigTemplates/formUtil.ts

@@ -18,6 +18,7 @@ export interface FormEntry {
     optional?: boolean;
     help?: string | null;
     type: string;
+    area?: boolean;
     value?: any;
     validators?: TemplateVariableValidator[];
 }
@@ -37,7 +38,8 @@ export function metadataFormEntryDescription(what: string): FormEntry {
         name: "_meta_description",
         label: "Description",
         optional: true,
-        type: "textarea",
+        type: "text",
+        area: true,
         help: `Provide some notes to yourself about this ${what} - perhaps to remind you how it is configured, where it stores the data, etc..`,
     };
 }
@@ -54,6 +56,7 @@ export function templateVariableFormEntry(variable: TemplateVariable, variableVa
         const defaultValue = variable.default ?? "";
         return {
             type: "text",
+            area: variable.multiline || false,
             value: variableValue == undefined ? defaultValue : variableValue,
             ...common_fields,
         };
@@ -87,7 +90,8 @@ export function templateSecretFormEntry(secret: TemplateSecret): FormEntry {
     return {
         name: secret.name,
         label: secret.label ?? secret.name,
-        type: "password",
+        type: secret.multiline ? "text" : "password",
+        area: secret.multiline || false,
         help: markup(secret.help || "", true),
         value: "",
         optional: secret.optional || false,

lib/galaxy/files/sources/ssh.py

@@ -1,70 +1,102 @@
-try:
-    from fs.sshfs.sshfs import SSHFS
-except ImportError:
-    SSHFS = None
-
+from io import StringIO
 from typing import (
     Optional,
+    TYPE_CHECKING,
     Union,
 )
 
-from galaxy.files.models import (
-    BaseFileSourceConfiguration,
-    BaseFileSourceTemplateConfiguration,
-    FilesSourceRuntimeContext,
+try:
+    from fsspec.implementations.sftp import SFTPFileSystem
+    from paramiko.ecdsakey import ECDSAKey
+    from paramiko.ed25519key import Ed25519Key
+    from paramiko.rsakey import RSAKey
+except ImportError:
+    SFTPFileSystem = None
+    if TYPE_CHECKING:
+        from paramiko.ecdsakey import ECDSAKey
+        from paramiko.ed25519key import Ed25519Key
+        from paramiko.rsakey import RSAKey
+
+from galaxy.exceptions import AuthenticationFailed
+from galaxy.files.models import FilesSourceRuntimeContext
+from galaxy.files.sources._fsspec import (
+    CacheOptionsDictType,
+    FsspecBaseFileSourceConfiguration,
+    FsspecBaseFileSourceTemplateConfiguration,
+    FsspecFilesSource,
 )
 from galaxy.util.config_templates import TemplateExpansion
-from ._pyfilesystem2 import PyFilesystem2FilesSource
 
 
-class SshFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
+def _parse_private_key(private_key: str, password: Optional[str]):
+    # Paramiko cannot autodetect the key type, so try the supported key classes.
+    for pkey_class in (RSAKey, ECDSAKey, Ed25519Key):
+        try:
+            with StringIO(private_key) as pkey_file:
+                return pkey_class.from_private_key(pkey_file, password=password)
+        except Exception:
+            continue
+
+    return None
+
+
+class SshFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration):
     host: Union[str, TemplateExpansion]
     user: Optional[Union[str, TemplateExpansion]] = None
     passwd: Optional[Union[str, TemplateExpansion]] = None
     pkey: Optional[Union[str, TemplateExpansion]] = None
     timeout: Union[int, TemplateExpansion] = 10
     port: Union[int, TemplateExpansion] = 22
     compress: Union[bool, TemplateExpansion] = False
-    config_path: Union[str, TemplateExpansion] = "~/.ssh/config"
     path: Union[str, TemplateExpansion]
 
 
-class SshFileSourceConfiguration(BaseFileSourceConfiguration):
+class SshFileSourceConfiguration(FsspecBaseFileSourceConfiguration):
     host: str
     user: Optional[str] = None
     passwd: Optional[str] = None
     pkey: Optional[str] = None
     timeout: int = 10
     port: int = 22
     compress: bool = False
-    config_path: str = "~/.ssh/config"
     path: str
 
 
-class SshFilesSource(PyFilesystem2FilesSource[SshFileSourceTemplateConfiguration, SshFileSourceConfiguration]):
+class SshFilesSource(FsspecFilesSource[SshFileSourceTemplateConfiguration, SshFileSourceConfiguration]):
     plugin_type = "ssh"
-    required_module = SSHFS
-    required_package = "fs.sshfs"
+    required_module = SFTPFileSystem
+    required_package = "fsspec"
 
     template_config_class = SshFileSourceTemplateConfiguration
     resolved_config_class = SshFileSourceConfiguration
 
-    def _open_fs(self, context: FilesSourceRuntimeContext[SshFileSourceConfiguration]):
-        if SSHFS is None:
+    def _open_fs(
+        self,
+        context: FilesSourceRuntimeContext[SshFileSourceConfiguration],
+        cache_options: CacheOptionsDictType,  # Ignored because fsspec's SFTPFileSystem does not support caching options.
+    ):
+        if SFTPFileSystem is None:
             raise self.required_package_exception
 
         config = context.config
-        handle = SSHFS(
+        pkey = None
+        password = config.passwd
+        if config.pkey:
+            pkey = _parse_private_key(config.pkey, config.passwd)
+            if pkey is None:
+                raise AuthenticationFailed("Invalid or unsupported SSH private key provided.")
+            password = None
+
+        fs = SFTPFileSystem(
             host=config.host,
-            user=config.user,
-            passwd=config.passwd,
-            pkey=config.pkey,
+            username=config.user,
+            password=password,
+            pkey=pkey,
             port=config.port,
             timeout=config.timeout,
             compress=config.compress,
-            config_path=config.config_path,
         )
-        return handle
+        return fs
 
     def _to_filesystem_path(self, path: str, config: SshFileSourceConfiguration) -> str:
         base = config.path.rstrip("/")

lib/galaxy/files/templates/examples/ssh.yml

@@ -0,0 +1,52 @@
+- id: ssh
+  version: 0
+  name: SSH Server
+  description: |
+    Setup connections to SSH servers to download and upload files.
+  configuration:
+    type: ssh
+    host: "{{ variables.host }}"
+    user: "{{ variables.user }}"
+    passwd: "{{ secrets.password }}"
+    pkey: "{{ secrets.pkey }}"
+    port: "{{ variables.port }}"
+    path: "{{ variables.path }}"
+    writable: "{{ variables.writable }}"
+  variables:
+    host:
+      label: SSH Host
+      type: string
+      help: Host of SSH Server to connect to.
+    user:
+      label: SSH User
+      type: string
+      optional: true
+      help: |
+        Username to connect with.
+    path:
+      label: Path
+      type: string
+      help: |
+        A path that is readable / writable by the user
+    writable:
+      label: Writable?
+      type: boolean
+      optional: true
+      help: Allow Galaxy to write data to this file source. Requires that user is allowed to write to this path on the host.
+    port:
+      label: SSH Port
+      type: integer
+      optional: true
+      help: Port used to connect to the SSH server.
+  secrets:
+    password:
+      label: SSH Password
+      optional: true
+      help: |
+        Password to connect to SSH server with.
+    pkey:
+      label: SSH private key
+      optional: true
+      multiline: true
+      help: |
+        Private key to be used. For encrypted keys the given password is used to decrypt.