Merge branch 'master' into fix/module-system-code-issues

jorgee · web-flow · commit 8aefdb2a421a · 2026-03-17T13:15:57.000+01:00
Signed-off-by: Jorge Ejarque &lt;jorgee@users.noreply.github.com&gt;
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/Architecture.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/Architecture.groovy
@@ -18,90 +18,117 @@ package nextflow.processor
 
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
-import groovy.transform.ToString
+import groovy.transform.Memoized
 
 /**
  * Implements the {@code arch} process directive, to hold information on the
  * CPU (micro)architecture required by the process.
  *
+ * <p>Supports multiple comma-separated architectures (e.g. {@code arch 'linux/amd64,linux/arm64'}).
+ * Multi-arch is fully supported by selected executors (e.g. Seqera) via {@link #platforms()} and
+ * {@link #containerPlatform()}. Other executors use {@link #getDockerArch()} and {@link #getSpackArch()},
+ * which return only the first (primary) architecture.
+ *
  * @author Marco De La Pierre <marco.delapierre@gmail.com>
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
  */
-@ToString
 @EqualsAndHashCode
 @CompileStatic
 class Architecture {
 
-/*
- * example of notation in process: arch 'linux/x86_64', target: 'haswell'
- * example of notation in config:  arch = [name: 'linux/x86_64', target: 'haswell']
- *
- * where dockerArch = 'linux/x86_64'
- *       spackArch = target ?: arch  // plus some validation for Spack syntax
- *
- *       platform = 'linux'
- *       arch = 'x86_64'
- *       target = 'haswell'
- *
- * [alternate example: 'arch linux/arm/v8', where platform = 'linux' and arch = 'arm/v8']
- */
-    // used in Nextflow
-    final String dockerArch
-    final String spackArch
-
-    // defined, but currently not used
-    final String platform
-    final String arch
-    final String target
-
-    static protected String getPlatform( String value ) {
-        // return value.minus(~'/.*') // keeping for reference
-        final chunks = value.tokenize('/')
-        if( chunks.size() > 1 )
-            return chunks[0]
-        else
-            return null
+    @EqualsAndHashCode
+    @CompileStatic
+    static class ArchEntry {
+        final String value
+
+        static protected String normalize( String name ) {
+            def chunks = name.tokenize('/')
+            if( chunks.size() == 3 )
+                return chunks[1] + '/' + chunks[2]
+            else if( chunks.size() == 2 )
+                return chunks[1]
+            else
+                return chunks[0]
+        }
+
+        static private void validate( String value, String name ) {
+            if( value == 'x86_64' || value == 'amd64' )
+                return
+            if( value == 'aarch64' || value == 'arm64' || value == 'arm64/v8' )
+                return
+            if( value == 'arm64/v7' )
+                return
+            throw new IllegalArgumentException("Not a valid `arch` value: ${name}")
+        }
+
+        static ArchEntry parse(String name) {
+            final value = normalize(name)
+            validate(value, name)
+            return new ArchEntry(value)
+        }
+
+        private ArchEntry(String value ) {
+            this.value = value
+        }
+
+        @Override
+        String toString() {
+            return value
+        }
     }
 
-    static protected String getArch( String value ) {
-        // return value.minus(~'.*/') // keeping for reference
-        def chunks = value.tokenize('/')
-        if( chunks.size() == 3 )
-            return chunks[1] + '/' + chunks[2]
-        else if( chunks.size() == 2 )
-            return chunks[1]
-        else
-            return chunks[0]
+    private final List<ArchEntry> entries
+
+    private final String target
+
+    /**
+     * @return all architectures as Docker platform strings (e.g. {@code ['linux/amd64', 'linux/arm64']})
+     */
+    List<String> platforms() {
+        entries.collect(it -> toDockerArch(it))
     }
 
-    static private String validateArchToDockerArch( Map res ) {
-        def value = getArch(res.name as String)
-        def name = res.name as String
+    /**
+     * @return all architectures as a comma-separated Docker platform string
+     *         (e.g. {@code 'linux/amd64,linux/arm64'})
+     */
+    String containerPlatform() {
+        platforms().join(',')
+    }
+
+    static private String toDockerArch(ArchEntry arch) {
+        final value = arch.value
         if( value == 'x86_64' || value == 'amd64' )
             return 'linux/amd64'
         if( value == 'aarch64' || value == 'arm64' || value == 'arm64/v8' )
             return 'linux/arm64'
         if( value == 'arm64/v7' )
             return 'linux/arm64/v7'
-        throw new IllegalArgumentException("Not a valid `arch` value: ${name}")
+        return null
+    }
+
+    /**
+     * @return the Docker platform string for the first (primary) architecture
+     */
+    @Memoized
+    String getDockerArch() {
+        return toDockerArch(entries[0])
     }
 
-    static private String validateArchToSpackArch( String value, String inputArch ) {
+    /**
+     * @return the Spack-compatible architecture name for the first (primary) architecture,
+     *         or the explicit {@code target} microarchitecture if specified
+     */
+    @Memoized
+    String getSpackArch() {
+        if( target != null )
+            return target
+        final value = entries[0].value
         if( value == 'x86_64' || value == 'amd64' )
             return 'x86_64'
         if( value == 'aarch64' || value == 'arm64' || value == 'arm64/v8' )
             return 'aarch64'
-        if( value == 'arm64/v7' )
-            return null
-        throw new IllegalArgumentException("Not a valid `arch` value: ${inputArch}")
-    }
-
-    static protected String getSpackArch( Map res ) {
-        if( res.target != null )
-            return res.target as String
-        else if( res.name != null )
-            return validateArchToSpackArch(getArch(res.name as String), res.name as String)
-        else
-            return null
+        return null
     }
 
     Architecture( String value ) {
@@ -112,14 +139,15 @@ class Architecture {
         if( !res.name )
             throw new IllegalArgumentException("Missing architecture `name` attribute")
 
-        this.dockerArch = validateArchToDockerArch(res)
-        this.platform = getPlatform(res.name as String)
-        this.arch = getArch(res.name as String)
+        this.target = res.target as String
+        this.entries = (res.name as String)
+            .tokenize(',')
+            .collect(it -> ArchEntry.parse(it.trim()) )
+    }
 
-        if( res.target != null )
-            this.target = res.target as String
-        if( res.name!=null || res.target!=null )
-            this.spackArch = getSpackArch(res)
+    @Override
+    String toString() {
+        return platforms().join(',')
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -24,7 +24,6 @@ import java.util.function.Function
 
 import com.google.common.hash.HashCode
 import groovy.transform.Memoized
-import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
 import nextflow.Session
 import nextflow.conda.CondaCache
@@ -41,7 +40,6 @@ import nextflow.exception.ProcessUnrecoverableException
 import nextflow.file.FileHelper
 import nextflow.file.FileHolder
 import nextflow.script.BodyDef
-import nextflow.script.ProcessConfigV1
 import nextflow.script.ProcessConfigV2
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
@@ -754,7 +752,7 @@ class TaskRun implements Cloneable {
 
     String getContainerPlatform() {
         final result = config.getArchitecture()
-        return result ? result.dockerArch : containerResolver().defaultContainerPlatform()
+        return result ? result.containerPlatform() : containerResolver().defaultContainerPlatform()
     }
 
     ResourcesBundle getModuleBundle() {
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/ArchitectureTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/ArchitectureTest.groovy
@@ -31,37 +31,94 @@ class ArchitectureTest extends Specification {
         when:
         def arch = new Architecture(VALUE)
         then:
-        arch.platform == PLAT
-        arch.arch == ARCH
-        arch.target == TAR
         arch.dockerArch == DOCK
         arch.spackArch == SPACK
 
         where:
-        VALUE                                  | PLAT     | ARCH        | TAR    | DOCK             | SPACK
-        'x86_64'                               | null     | 'x86_64'    | null   | 'linux/amd64'    | 'x86_64'
-        'linux/x86_64'                         | 'linux'  | 'x86_64'    | null   | 'linux/amd64'    | 'x86_64'
-        'amd64'                                | null     | 'amd64'     | null   | 'linux/amd64'    | 'x86_64'
-        'aarch64'                              | null     | 'aarch64'   | null   | 'linux/arm64'    | 'aarch64'
-        'arm64'                                | null     | 'arm64'     | null   | 'linux/arm64'    | 'aarch64'
-        'linux/arm64/v8'                       | 'linux'  | 'arm64/v8'  | null   | 'linux/arm64'    | 'aarch64'
-        'linux/arm64/v7'                       | 'linux'  | 'arm64/v7'  | null   | 'linux/arm64/v7' | null
+        VALUE                | DOCK             | SPACK
+        'x86_64'             | 'linux/amd64'    | 'x86_64'
+        'linux/x86_64'       | 'linux/amd64'    | 'x86_64'
+        'amd64'              | 'linux/amd64'    | 'x86_64'
+        'aarch64'            | 'linux/arm64'    | 'aarch64'
+        'arm64'              | 'linux/arm64'    | 'aarch64'
+        'linux/arm64/v8'     | 'linux/arm64'    | 'aarch64'
+        'linux/arm64/v7'     | 'linux/arm64/v7' | null
     }
 
     def 'should define arch with map' () {
         when:
         def arch = new Architecture(VALUE)
         then:
-        arch.platform == PLAT
-        arch.arch == ARCH
-        arch.target == TAR
         arch.dockerArch == DOCK
         arch.spackArch == SPACK
 
         where:
-        VALUE                                  | PLAT     | ARCH        | TAR    | DOCK             | SPACK
-        [name: 'amd64', target: 'zen3']        | null     | 'amd64'     | 'zen3' | 'linux/amd64'    | 'zen3'
-        [name: 'arm64', target: 'zen3']        | null     | 'arm64'     | 'zen3' | 'linux/arm64'    | 'zen3'
-        [name: 'linux/x86_64', target: 'zen3'] | 'linux'  | 'x86_64'    | 'zen3' | 'linux/amd64'    | 'zen3'
+        VALUE                                  | DOCK             | SPACK
+        [name: 'amd64', target: 'zen3']        | 'linux/amd64'    | 'zen3'
+        [name: 'arm64', target: 'zen3']        | 'linux/arm64'    | 'zen3'
+        [name: 'linux/x86_64', target: 'zen3'] | 'linux/amd64'    | 'zen3'
+    }
+
+    @Unroll
+    def 'should normalize arch from name' () {
+        expect:
+        Architecture.ArchEntry.normalize(NAME) == EXPECTED
+
+        where:
+        NAME             | EXPECTED
+        'x86_64'         | 'x86_64'
+        'linux/x86_64'   | 'x86_64'
+        'linux/arm64/v8' | 'arm64/v8'
+        'linux/arm64/v7' | 'arm64/v7'
+        'arm64'          | 'arm64'
+        'aarch64'        | 'aarch64'
+    }
+
+    def 'should return dockerArch as string representation' () {
+        expect:
+        new Architecture('linux/x86_64').toString() == 'linux/amd64'
+        new Architecture('arm64').toString() == 'linux/arm64'
+        new Architecture('linux/amd64,linux/arm64').toString() == 'linux/amd64,linux/arm64'
+    }
+
+    def 'should parse multi-arch value' () {
+        when:
+        def arch = new Architecture('linux/amd64,linux/arm64')
+        then:
+        arch.dockerArch == 'linux/amd64'
+        arch.spackArch == 'x86_64'
+        arch.platforms() == ['linux/amd64', 'linux/arm64']
+    }
+
+    def 'should parse multi-arch with spaces' () {
+        when:
+        def arch = new Architecture('linux/amd64, linux/arm64')
+        then:
+        arch.platforms() == ['linux/amd64', 'linux/arm64']
+    }
+
+    def 'should return containerPlatform as comma-separated string' () {
+        expect:
+        new Architecture('linux/amd64').containerPlatform() == 'linux/amd64'
+        new Architecture('linux/amd64,linux/arm64').containerPlatform() == 'linux/amd64,linux/arm64'
+    }
+
+    def 'should define multi-arch with map' () {
+        when:
+        def arch = new Architecture([name: 'amd64,arm64', target: 'zen3'])
+        then:
+        arch.dockerArch == 'linux/amd64'
+        arch.spackArch == 'zen3'
+        arch.platforms() == ['linux/amd64', 'linux/arm64']
+        arch.containerPlatform() == 'linux/amd64,linux/arm64'
+    }
+
+    def 'should use primary arch for dockerArch and spackArch with multi-arch' () {
+        when:
+        def arch = new Architecture('arm64,amd64')
+        then:
+        arch.dockerArch == 'linux/arm64'
+        arch.spackArch == 'aarch64'
+        arch.platforms() == ['linux/arm64', 'linux/amd64']
     }
 }
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy
@@ -616,7 +616,7 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler<String,Job
             final diskGb = task.config.getDisk()?.toGiga()?.toInteger() ?: 50
             container.ephemeralStorage( EphemeralStorage.builder().sizeInGiB(diskGb).build() )
             // check for arm64 cpu architecture
-            if( task.config.getArchitecture()?.arch == 'arm64' )
+            if( task.config.getArchitecture()?.dockerArch == 'linux/arm64' )
                 container.runtimePlatform(RuntimePlatform.builder().cpuArchitecture('ARM64').build())
         }
 
diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy
@@ -401,11 +401,22 @@ class WaveClient {
             : new URL(DEFAULT_S5CMD_AMD64_URL)
     }
 
+    protected static URL replaceFusionArch(URL url, String platform) {
+        final isArm = platform.tokenize('/')?.contains('arm64')
+        final targetArch = isArm ? 'arm64' : 'amd64'
+        final replaced = url.toString().replaceAll(/(?<=[-_])(amd64|arm64)(?=\.)/, targetArch)
+        return replaced != url.toString() ? new URL(replaced) : url
+    }
+
     ContainerConfig resolveContainerConfig(String platform = DEFAULT_DOCKER_PLATFORM) {
         final urls = new ArrayList<URL>(config.containerConfigUrl())
+        final platforms = platform ? platform.tokenize(',') : List.of(DEFAULT_DOCKER_PLATFORM)
         if( fusion.enabled() ) {
-            final fusionUrl = fusion.containerConfigUrl() ?: defaultFusionUrl(platform)
-            urls.add(fusionUrl)
+            final customUrl = fusion.containerConfigUrl()
+            for( String p : platforms ) {
+                final fusionUrl = customUrl ? replaceFusionArch(customUrl, p.trim()) : defaultFusionUrl(p.trim())
+                urls.add(fusionUrl)
+            }
         }
         if( awsFargate ) {
             final s5cmdUrl = s5cmdConfigUrl ?: defaultS5cmdUrl(platform)
@@ -520,7 +531,7 @@ class WaveClient {
         return resolveAssets0(attrs, bundle, singularity, dockerArch)
     }
 
-    protected WaveAssets resolveAssets0(Map<String,String> attrs, ResourcesBundle bundle, boolean singularity, String dockerArch) {
+    protected WaveAssets resolveAssets0(Map<String,String> attrs, ResourcesBundle bundle, boolean singularity, String platform) {
 
         final scriptType = singularity ? 'singularityfile' : 'dockerfile'
         String containerScript = attrs.get(scriptType)
@@ -581,11 +592,6 @@ class WaveClient {
                     ? projectResources(session.binDir)
                     : null
 
-        /*
-         * the container platform to be used
-         */
-        final platform = dockerArch
-
         // check is a valid container image
         WaveAssets.validateContainerName(containerImage)
         // read the container config and go ahead

Original file line number	Diff line number	Diff line change
`@@ -616,7 +616,7 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler<String,Job`
`616`	`616`	`final diskGb = task.config.getDisk()?.toGiga()?.toInteger() ?: 50`
`617`	`617`	`container.ephemeralStorage( EphemeralStorage.builder().sizeInGiB(diskGb).build() )`
`618`	`618`	`// check for arm64 cpu architecture`
`619`		`- if( task.config.getArchitecture()?.arch == 'arm64' )`
	`619`	`+ if( task.config.getArchitecture()?.dockerArch == 'linux/arm64' )`
`620`	`620`	`container.runtimePlatform(RuntimePlatform.builder().cpuArchitecture('ARM64').build())`
`621`	`621`	`}`
`622`	`622`