Skip to content

Commit c7ca369

Browse files
pditommasoclaudebentshermanramonamela
authored
Add multi-arch support to arch process directive (#6897)
* Add multi-arch support to `arch` process directive Allow comma-separated platform values in the `arch` directive (e.g. `arch 'linux/amd64,linux/arm64'`) to enable Wave to build multi-platform container images. Add `seqera.executor.provider` setting to specify the compute backend provider type (e.g. aws, local), used together with region to select the matching compute environment. Bump nf-seqera to 0.15.0 and sched-client to 0.41.0-SNAPSHOT. Refactor Architecture class to support comma-separated architectures (e.g. `arch 'linux/amd64,linux/arm64'`). Multi-arch is fully supported by selected executors (Seqera) via platforms()/containerPlatform(). Other executors use getDockerArch()/getSpackArch() which return only the first (primary) architecture. Also rename inner class Arch to ArchEntry for clarity. Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> --------- Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Ben Sherman <bentshermann@gmail.com> Co-authored-by: Ramon Amela Milian <25862624+ramonamela@users.noreply.github.com>
1 parent a0ac594 commit c7ca369

5 files changed

Lines changed: 183 additions & 94 deletions

File tree

‎modules/nextflow/src/main/groovy/nextflow/processor/Architecture.groovy‎

Lines changed: 92 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -18,90 +18,117 @@ package nextflow.processor
1818

1919
import groovy.transform.CompileStatic
2020
import groovy.transform.EqualsAndHashCode
21-
import groovy.transform.ToString
21+
import groovy.transform.Memoized
2222

2323
/**
2424
* Implements the {@code arch} process directive, to hold information on the
2525
* CPU (micro)architecture required by the process.
2626
*
27+
* <p>Supports multiple comma-separated architectures (e.g. {@code arch 'linux/amd64,linux/arm64'}).
28+
* Multi-arch is fully supported by selected executors (e.g. Seqera) via {@link #platforms()} and
29+
* {@link #containerPlatform()}. Other executors use {@link #getDockerArch()} and {@link #getSpackArch()},
30+
* which return only the first (primary) architecture.
31+
*
2732
* @author Marco De La Pierre <marco.delapierre@gmail.com>
33+
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
2834
*/
29-
@ToString
3035
@EqualsAndHashCode
3136
@CompileStatic
3237
class Architecture {
3338

34-
/*
35-
* example of notation in process: arch 'linux/x86_64', target: 'haswell'
36-
* example of notation in config: arch = [name: 'linux/x86_64', target: 'haswell']
37-
*
38-
* where dockerArch = 'linux/x86_64'
39-
* spackArch = target ?: arch // plus some validation for Spack syntax
40-
*
41-
* platform = 'linux'
42-
* arch = 'x86_64'
43-
* target = 'haswell'
44-
*
45-
* [alternate example: 'arch linux/arm/v8', where platform = 'linux' and arch = 'arm/v8']
46-
*/
47-
// used in Nextflow
48-
final String dockerArch
49-
final String spackArch
50-
51-
// defined, but currently not used
52-
final String platform
53-
final String arch
54-
final String target
55-
56-
static protected String getPlatform( String value ) {
57-
// return value.minus(~'/.*') // keeping for reference
58-
final chunks = value.tokenize('/')
59-
if( chunks.size() > 1 )
60-
return chunks[0]
61-
else
62-
return null
39+
@EqualsAndHashCode
40+
@CompileStatic
41+
static class ArchEntry {
42+
final String value
43+
44+
static protected String normalize( String name ) {
45+
def chunks = name.tokenize('/')
46+
if( chunks.size() == 3 )
47+
return chunks[1] + '/' + chunks[2]
48+
else if( chunks.size() == 2 )
49+
return chunks[1]
50+
else
51+
return chunks[0]
52+
}
53+
54+
static private void validate( String value, String name ) {
55+
if( value == 'x86_64' || value == 'amd64' )
56+
return
57+
if( value == 'aarch64' || value == 'arm64' || value == 'arm64/v8' )
58+
return
59+
if( value == 'arm64/v7' )
60+
return
61+
throw new IllegalArgumentException("Not a valid `arch` value: ${name}")
62+
}
63+
64+
static ArchEntry parse(String name) {
65+
final value = normalize(name)
66+
validate(value, name)
67+
return new ArchEntry(value)
68+
}
69+
70+
private ArchEntry(String value ) {
71+
this.value = value
72+
}
73+
74+
@Override
75+
String toString() {
76+
return value
77+
}
6378
}
6479

65-
static protected String getArch( String value ) {
66-
// return value.minus(~'.*/') // keeping for reference
67-
def chunks = value.tokenize('/')
68-
if( chunks.size() == 3 )
69-
return chunks[1] + '/' + chunks[2]
70-
else if( chunks.size() == 2 )
71-
return chunks[1]
72-
else
73-
return chunks[0]
80+
private final List<ArchEntry> entries
81+
82+
private final String target
83+
84+
/**
85+
* @return all architectures as Docker platform strings (e.g. {@code ['linux/amd64', 'linux/arm64']})
86+
*/
87+
List<String> platforms() {
88+
entries.collect(it -> toDockerArch(it))
7489
}
7590

76-
static private String validateArchToDockerArch( Map res ) {
77-
def value = getArch(res.name as String)
78-
def name = res.name as String
91+
/**
92+
* @return all architectures as a comma-separated Docker platform string
93+
* (e.g. {@code 'linux/amd64,linux/arm64'})
94+
*/
95+
String containerPlatform() {
96+
platforms().join(',')
97+
}
98+
99+
static private String toDockerArch(ArchEntry arch) {
100+
final value = arch.value
79101
if( value == 'x86_64' || value == 'amd64' )
80102
return 'linux/amd64'
81103
if( value == 'aarch64' || value == 'arm64' || value == 'arm64/v8' )
82104
return 'linux/arm64'
83105
if( value == 'arm64/v7' )
84106
return 'linux/arm64/v7'
85-
throw new IllegalArgumentException("Not a valid `arch` value: ${name}")
107+
return null
108+
}
109+
110+
/**
111+
* @return the Docker platform string for the first (primary) architecture
112+
*/
113+
@Memoized
114+
String getDockerArch() {
115+
return toDockerArch(entries[0])
86116
}
87117

88-
static private String validateArchToSpackArch( String value, String inputArch ) {
118+
/**
119+
* @return the Spack-compatible architecture name for the first (primary) architecture,
120+
* or the explicit {@code target} microarchitecture if specified
121+
*/
122+
@Memoized
123+
String getSpackArch() {
124+
if( target != null )
125+
return target
126+
final value = entries[0].value
89127
if( value == 'x86_64' || value == 'amd64' )
90128
return 'x86_64'
91129
if( value == 'aarch64' || value == 'arm64' || value == 'arm64/v8' )
92130
return 'aarch64'
93-
if( value == 'arm64/v7' )
94-
return null
95-
throw new IllegalArgumentException("Not a valid `arch` value: ${inputArch}")
96-
}
97-
98-
static protected String getSpackArch( Map res ) {
99-
if( res.target != null )
100-
return res.target as String
101-
else if( res.name != null )
102-
return validateArchToSpackArch(getArch(res.name as String), res.name as String)
103-
else
104-
return null
131+
return null
105132
}
106133

107134
Architecture( String value ) {
@@ -112,14 +139,15 @@ class Architecture {
112139
if( !res.name )
113140
throw new IllegalArgumentException("Missing architecture `name` attribute")
114141

115-
this.dockerArch = validateArchToDockerArch(res)
116-
this.platform = getPlatform(res.name as String)
117-
this.arch = getArch(res.name as String)
142+
this.target = res.target as String
143+
this.entries = (res.name as String)
144+
.tokenize(',')
145+
.collect(it -> ArchEntry.parse(it.trim()) )
146+
}
118147

119-
if( res.target != null )
120-
this.target = res.target as String
121-
if( res.name!=null || res.target!=null )
122-
this.spackArch = getSpackArch(res)
148+
@Override
149+
String toString() {
150+
return platforms().join(',')
123151
}
124152

125153
}

‎modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import java.util.function.Function
2424

2525
import com.google.common.hash.HashCode
2626
import groovy.transform.Memoized
27-
import groovy.transform.Memoized
2827
import groovy.util.logging.Slf4j
2928
import nextflow.Session
3029
import nextflow.conda.CondaCache
@@ -41,7 +40,6 @@ import nextflow.exception.ProcessUnrecoverableException
4140
import nextflow.file.FileHelper
4241
import nextflow.file.FileHolder
4342
import nextflow.script.BodyDef
44-
import nextflow.script.ProcessConfigV1
4543
import nextflow.script.ProcessConfigV2
4644
import nextflow.script.ScriptType
4745
import nextflow.script.TaskClosure
@@ -754,7 +752,7 @@ class TaskRun implements Cloneable {
754752

755753
String getContainerPlatform() {
756754
final result = config.getArchitecture()
757-
return result ? result.dockerArch : containerResolver().defaultContainerPlatform()
755+
return result ? result.containerPlatform() : containerResolver().defaultContainerPlatform()
758756
}
759757

760758
ResourcesBundle getModuleBundle() {

‎modules/nextflow/src/test/groovy/nextflow/processor/ArchitectureTest.groovy‎

Lines changed: 75 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -31,37 +31,94 @@ class ArchitectureTest extends Specification {
3131
when:
3232
def arch = new Architecture(VALUE)
3333
then:
34-
arch.platform == PLAT
35-
arch.arch == ARCH
36-
arch.target == TAR
3734
arch.dockerArch == DOCK
3835
arch.spackArch == SPACK
3936

4037
where:
41-
VALUE | PLAT | ARCH | TAR | DOCK | SPACK
42-
'x86_64' | null | 'x86_64' | null | 'linux/amd64' | 'x86_64'
43-
'linux/x86_64' | 'linux' | 'x86_64' | null | 'linux/amd64' | 'x86_64'
44-
'amd64' | null | 'amd64' | null | 'linux/amd64' | 'x86_64'
45-
'aarch64' | null | 'aarch64' | null | 'linux/arm64' | 'aarch64'
46-
'arm64' | null | 'arm64' | null | 'linux/arm64' | 'aarch64'
47-
'linux/arm64/v8' | 'linux' | 'arm64/v8' | null | 'linux/arm64' | 'aarch64'
48-
'linux/arm64/v7' | 'linux' | 'arm64/v7' | null | 'linux/arm64/v7' | null
38+
VALUE | DOCK | SPACK
39+
'x86_64' | 'linux/amd64' | 'x86_64'
40+
'linux/x86_64' | 'linux/amd64' | 'x86_64'
41+
'amd64' | 'linux/amd64' | 'x86_64'
42+
'aarch64' | 'linux/arm64' | 'aarch64'
43+
'arm64' | 'linux/arm64' | 'aarch64'
44+
'linux/arm64/v8' | 'linux/arm64' | 'aarch64'
45+
'linux/arm64/v7' | 'linux/arm64/v7' | null
4946
}
5047

5148
def 'should define arch with map' () {
5249
when:
5350
def arch = new Architecture(VALUE)
5451
then:
55-
arch.platform == PLAT
56-
arch.arch == ARCH
57-
arch.target == TAR
5852
arch.dockerArch == DOCK
5953
arch.spackArch == SPACK
6054

6155
where:
62-
VALUE | PLAT | ARCH | TAR | DOCK | SPACK
63-
[name: 'amd64', target: 'zen3'] | null | 'amd64' | 'zen3' | 'linux/amd64' | 'zen3'
64-
[name: 'arm64', target: 'zen3'] | null | 'arm64' | 'zen3' | 'linux/arm64' | 'zen3'
65-
[name: 'linux/x86_64', target: 'zen3'] | 'linux' | 'x86_64' | 'zen3' | 'linux/amd64' | 'zen3'
56+
VALUE | DOCK | SPACK
57+
[name: 'amd64', target: 'zen3'] | 'linux/amd64' | 'zen3'
58+
[name: 'arm64', target: 'zen3'] | 'linux/arm64' | 'zen3'
59+
[name: 'linux/x86_64', target: 'zen3'] | 'linux/amd64' | 'zen3'
60+
}
61+
62+
@Unroll
63+
def 'should normalize arch from name' () {
64+
expect:
65+
Architecture.ArchEntry.normalize(NAME) == EXPECTED
66+
67+
where:
68+
NAME | EXPECTED
69+
'x86_64' | 'x86_64'
70+
'linux/x86_64' | 'x86_64'
71+
'linux/arm64/v8' | 'arm64/v8'
72+
'linux/arm64/v7' | 'arm64/v7'
73+
'arm64' | 'arm64'
74+
'aarch64' | 'aarch64'
75+
}
76+
77+
def 'should return dockerArch as string representation' () {
78+
expect:
79+
new Architecture('linux/x86_64').toString() == 'linux/amd64'
80+
new Architecture('arm64').toString() == 'linux/arm64'
81+
new Architecture('linux/amd64,linux/arm64').toString() == 'linux/amd64,linux/arm64'
82+
}
83+
84+
def 'should parse multi-arch value' () {
85+
when:
86+
def arch = new Architecture('linux/amd64,linux/arm64')
87+
then:
88+
arch.dockerArch == 'linux/amd64'
89+
arch.spackArch == 'x86_64'
90+
arch.platforms() == ['linux/amd64', 'linux/arm64']
91+
}
92+
93+
def 'should parse multi-arch with spaces' () {
94+
when:
95+
def arch = new Architecture('linux/amd64, linux/arm64')
96+
then:
97+
arch.platforms() == ['linux/amd64', 'linux/arm64']
98+
}
99+
100+
def 'should return containerPlatform as comma-separated string' () {
101+
expect:
102+
new Architecture('linux/amd64').containerPlatform() == 'linux/amd64'
103+
new Architecture('linux/amd64,linux/arm64').containerPlatform() == 'linux/amd64,linux/arm64'
104+
}
105+
106+
def 'should define multi-arch with map' () {
107+
when:
108+
def arch = new Architecture([name: 'amd64,arm64', target: 'zen3'])
109+
then:
110+
arch.dockerArch == 'linux/amd64'
111+
arch.spackArch == 'zen3'
112+
arch.platforms() == ['linux/amd64', 'linux/arm64']
113+
arch.containerPlatform() == 'linux/amd64,linux/arm64'
114+
}
115+
116+
def 'should use primary arch for dockerArch and spackArch with multi-arch' () {
117+
when:
118+
def arch = new Architecture('arm64,amd64')
119+
then:
120+
arch.dockerArch == 'linux/arm64'
121+
arch.spackArch == 'aarch64'
122+
arch.platforms() == ['linux/arm64', 'linux/amd64']
66123
}
67124
}

‎plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler<String,Job
616616
final diskGb = task.config.getDisk()?.toGiga()?.toInteger() ?: 50
617617
container.ephemeralStorage( EphemeralStorage.builder().sizeInGiB(diskGb).build() )
618618
// check for arm64 cpu architecture
619-
if( task.config.getArchitecture()?.arch == 'arm64' )
619+
if( task.config.getArchitecture()?.dockerArch == 'linux/arm64' )
620620
container.runtimePlatform(RuntimePlatform.builder().cpuArchitecture('ARM64').build())
621621
}
622622

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -401,11 +401,22 @@ class WaveClient {
401401
: new URL(DEFAULT_S5CMD_AMD64_URL)
402402
}
403403

404+
protected static URL replaceFusionArch(URL url, String platform) {
405+
final isArm = platform.tokenize('/')?.contains('arm64')
406+
final targetArch = isArm ? 'arm64' : 'amd64'
407+
final replaced = url.toString().replaceAll(/(?<=[-_])(amd64|arm64)(?=\.)/, targetArch)
408+
return replaced != url.toString() ? new URL(replaced) : url
409+
}
410+
404411
ContainerConfig resolveContainerConfig(String platform = DEFAULT_DOCKER_PLATFORM) {
405412
final urls = new ArrayList<URL>(config.containerConfigUrl())
413+
final platforms = platform ? platform.tokenize(',') : List.of(DEFAULT_DOCKER_PLATFORM)
406414
if( fusion.enabled() ) {
407-
final fusionUrl = fusion.containerConfigUrl() ?: defaultFusionUrl(platform)
408-
urls.add(fusionUrl)
415+
final customUrl = fusion.containerConfigUrl()
416+
for( String p : platforms ) {
417+
final fusionUrl = customUrl ? replaceFusionArch(customUrl, p.trim()) : defaultFusionUrl(p.trim())
418+
urls.add(fusionUrl)
419+
}
409420
}
410421
if( awsFargate ) {
411422
final s5cmdUrl = s5cmdConfigUrl ?: defaultS5cmdUrl(platform)
@@ -520,7 +531,7 @@ class WaveClient {
520531
return resolveAssets0(attrs, bundle, singularity, dockerArch)
521532
}
522533

523-
protected WaveAssets resolveAssets0(Map<String,String> attrs, ResourcesBundle bundle, boolean singularity, String dockerArch) {
534+
protected WaveAssets resolveAssets0(Map<String,String> attrs, ResourcesBundle bundle, boolean singularity, String platform) {
524535

525536
final scriptType = singularity ? 'singularityfile' : 'dockerfile'
526537
String containerScript = attrs.get(scriptType)
@@ -581,11 +592,6 @@ class WaveClient {
581592
? projectResources(session.binDir)
582593
: null
583594

584-
/*
585-
* the container platform to be used
586-
*/
587-
final platform = dockerArch
588-
589595
// check is a valid container image
590596
WaveAssets.validateContainerName(containerImage)
591597
// read the container config and go ahead

0 commit comments

Comments
 (0)
⚡