Add Support for HTTP Headers in URL Fetch Requests with Secure Storage for Landing Requests

client/src/api/schema/schema.ts

@@ -23479,6 +23479,13 @@ export interface components {
             extra_files?: components["schemas"]["ExtraFiles"] | null;
             /** Hashes */
             hashes?: components["schemas"]["FetchDatasetHash"][] | null;
+            /**
+             * Headers
+             * @description Optional headers to include in the URL fetch request
+             */
+            headers?: {
+                [key: string]: string;
+            } | null;
             /**
              * Info
              * @description Free text field that can be used to store arbitrary information about the dataset. This used to be prominently

config/url_headers_conf.yml.sample

@@ -0,0 +1 @@
+../lib/galaxy/config/sample/url_headers_conf.yml.sample

doc/source/admin/galaxy_options.rst

@@ -5638,6 +5638,26 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``url_headers_config_file``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Configuration file for URL request headers allow-list with URL
+    pattern matching. This file defines which HTTP headers are allowed
+    in URL fetch requests based on URL patterns, and whether they
+    should be treated as sensitive (encrypted in the vault) or not. If
+    no allow-list is specified, no headers will be allowed in URL
+    requests. This provides fine-grained security control over what
+    headers can be sent when Galaxy fetches external URLs on behalf of
+    users, allowing different headers for different target domains or
+    services.
+    The value of this option will be resolved with respect to
+    <config_dir>.
+:Default: ``url_headers_conf.yml``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``display_builtin_converters``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -291,6 +291,7 @@ def __init__(self, **kwargs):
         self.monitor_thread_join_timeout = 1
         self.integrated_tool_panel_config = None
         self.vault_config_file = kwargs.get("vault_config_file")
+        self.url_headers_config_file = None
         self.max_discovered_files = 10000
         self.display_builtin_converters = True
         self.enable_notification_system = True

lib/galaxy/config/sample/galaxy.yml.sample

@@ -3048,6 +3048,18 @@ galaxy:
   # <config_dir>.
   #vault_config_file: vault_conf.yml
 
+  # Configuration file for URL request headers allow-list with URL
+  # pattern matching. This file defines which HTTP headers are allowed
+  # in URL fetch requests based on URL patterns, and whether they should
+  # be treated as sensitive (encrypted in the vault) or not. If no
+  # allow-list is specified, no headers will be allowed in URL requests.
+  # This provides fine-grained security control over what headers can be
+  # sent when Galaxy fetches external URLs on behalf of users, allowing
+  # different headers for different target domains or services.
+  # The value of this option will be resolved with respect to
+  # <config_dir>.
+  #url_headers_config_file: url_headers_conf.yml
+
   # Display built-in converters in the tool panel.
   #display_builtin_converters: true
 

lib/galaxy/config/sample/url_headers_conf.yml.sample

@@ -0,0 +1,115 @@
+# Allowed URL Headers Configuration
+#
+# This file defines which HTTP headers are allowed in URL fetch requests based
+# on URL patterns, and whether they should be treated as sensitive (encrypted
+# in the vault) or not.
+#
+# If no allow-list is specified or this file is empty/missing, NO headers will
+# be allowed in URL requests.
+#
+# Configuration structure:
+# patterns:
+#   - url_pattern: A regular expression pattern to match URLs
+#     headers:
+#       - name: The exact header name (case-insensitive)
+#         sensitive: Whether this header contains sensitive information that should
+#                    be encrypted when stored in the database (requires vault configuration)
+#
+# IMPORTANT:
+# ------------------------------------
+# When a URL matches MULTIPLE patterns, the union of all allowed headers is used.
+# This means you can compose permissions from multiple patterns for flexibility.
+#
+# Example: A URL matching both pattern A (allows headers X, Y) and pattern B
+# (allows headers Y, Z) will allow headers X, Y, and Z.
+#
+# Security: If ANY matching pattern marks a header as sensitive, it will be
+# treated as sensitive (secure-by-default).
+#
+# The following examples are for illustration purposes only; please use only the minimum configuration for your needs.
+# Examples:
+
+patterns:
+  # GitHub API access - allow authentication headers for GitHub URLs
+  - url_pattern: "^https://api\\.github\\.com/.*"
+    headers:
+      - name: Authorization
+        sensitive: true
+      - name: Accept
+        sensitive: false
+      - name: X-GitHub-Api-Version
+        sensitive: false
+
+  # Generic GitHub content (raw files, releases) - no auth needed
+  - url_pattern: "^https://(raw\\.githubusercontent\\.com|github\\.com/.*/releases/download)/.*"
+    headers:
+      - name: Accept
+        sensitive: false
+      - name: Accept-Encoding
+        sensitive: false
+
+  # AWS S3 buckets - allow AWS authentication headers
+  - url_pattern: "^https://.*\\.s3\\..+\\.amazonaws\\.com/.*"
+    headers:
+      - name: Authorization
+        sensitive: true
+      - name: X-Amz-Date
+        sensitive: false
+      - name: X-Amz-Content-Sha256
+        sensitive: false
+      - name: X-Amz-Security-Token
+        sensitive: true
+
+  # Generic cloud storage APIs
+  - url_pattern: "^https://.*\\.(googleapis\\.com|azure\\.com|digitaloceanspaces\\.com)/.*"
+    headers:
+      - name: Authorization
+        sensitive: true
+      - name: X-API-Key
+        sensitive: true
+      - name: Accept
+        sensitive: false
+
+  # FTP over HTTP services
+  - url_pattern: "^https?://ftp\\..*/.*"
+    headers:
+      - name: Authorization
+        sensitive: true
+      - name: Accept
+        sensitive: false
+
+  # Academic/research data repositories
+  - url_pattern: "^https://.*(zenodo\\.org|figshare\\.com|dryad\\.org|dataverse\\.org)/.*"
+    headers:
+      - name: Authorization
+        sensitive: true
+      - name: X-API-Key
+        sensitive: true
+      - name: Accept
+        sensitive: false
+
+  # HTTPS URLs - basic headers only (most restrictive for unknown sources)
+  - url_pattern: "^https://.*"
+    headers:
+      - name: Authorization
+        sensitive: true
+      - name: X-Auth-Token
+        sensitive: true
+      - name: X-API-Key
+        sensitive: true
+      - name: Accept
+        sensitive: false
+      - name: Accept-Language
+        sensitive: false
+      - name: Accept-Encoding
+        sensitive: false
+      - name: Cache-Control
+        sensitive: false
+
+# Security notes:
+# - All matching patterns contribute their allowed headers (union of permissions)
+# - If ANY pattern marks a header as sensitive, it's treated as sensitive
+# - Only add headers that are absolutely necessary for your use case
+# - When in doubt, mark headers as sensitive to ensure encryption
+# - Patterns are order-independent, making configuration more composable
+# - HTTP (non-HTTPS) URLs are generally not recommended and may be blocked

lib/galaxy/config/schemas/config_schema.yml

@@ -4162,6 +4162,20 @@ mapping:
         desc: |
           Vault config file.
 
+      url_headers_config_file:
+        type: str
+        default: url_headers_conf.yml
+        path_resolves_to: config_dir
+        required: false
+        desc: |
+          Configuration file for URL request headers allow-list with URL pattern matching.
+          This file defines which HTTP headers are allowed in URL fetch requests based
+          on URL patterns, and whether they should be treated as sensitive (encrypted
+          in the vault) or not. If no allow-list is specified, no headers will be
+          allowed in URL requests. This provides fine-grained security control over
+          what headers can be sent when Galaxy fetches external URLs on behalf of users,
+          allowing different headers for different target domains or services.
+
       display_builtin_converters:
         type: bool
         default: true