Merge branch 'master' into add-origPath-to-FileHolder

Author: bentsherman

.github/stale.yml

@@ -84,6 +84,8 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_S3FS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_S3FS_SECRET_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           NXF_BITBUCKET_ACCESS_TOKEN: ${{ secrets.NXF_BITBUCKET_ACCESS_TOKEN }}
           NXF_GITHUB_ACCESS_TOKEN: ${{ secrets.NXF_GITHUB_ACCESS_TOKEN }}
           NXF_GITLAB_ACCESS_TOKEN: ${{ secrets.NXF_GITLAB_ACCESS_TOKEN }}
@@ -95,6 +97,7 @@ jobs:
           AZURE_BATCH_ACCOUNT_KEY: ${{ secrets.AZURE_BATCH_ACCOUNT_KEY }}
 
       - name: Publish tests report
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: report-unit-tests-jdk-${{ matrix.java_version }}

.github/workflows/build.yml

@@ -0,0 +1,21 @@
+name: 'Mark stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-stale: 180
+          days-before-close: -1
+          stale-issue-label: stale
+          stale-issue-message: ''
+          stale-pr-label: stale
+          stale-pr-message: ''
+          exempt-issue-labels: bug,planned,security
+          exempt-all-milestones: true
+          exempt-all-assignees: true

.github/workflows/stale.yml

@@ -1 +1 @@
-25.03.1-edge
+25.06.0-edge

VERSION

@@ -15,7 +15,6 @@
  */
 
 plugins {
-    id "io.codearte.nexus-staging" version "0.30.0"
     id 'java'
     id 'idea'
 }
@@ -103,8 +102,8 @@ allprojects {
 
         // Documentation required libraries
         groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
-        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.26"
-        groovyDoc "org.apache.groovy:groovy-ant:4.0.26"
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.28"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.28"
     }
 
     test {
@@ -268,8 +267,8 @@ task exportClasspath {
     }
 }
 
-ext.nexusUsername = project.findProperty('nexusUsername')
-ext.nexusPassword = project.findProperty('nexusPassword')
+ext.nexusUsername = project.findProperty('nexusUsername') ?: System.getenv('AWS_ACCESS_KEY_ID')
+ext.nexusPassword = project.findProperty('nexusPassword') ?: System.getenv('AWS_SECRET_ACCESS_KEY')
 ext.nexusFullName = project.findProperty('nexusFullName')
 ext.nexusEmail = project.findProperty('nexusEmail')
 
@@ -347,12 +346,12 @@ configure(coreProjects) {
         repositories {
             maven {
                 // change URLs to point to your repos, e.g. http://my.org/repo
-                def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
-                def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/"
+                def releasesRepoUrl = "s3://maven.seqera.io/releases/"
+                def snapshotsRepoUrl = "s3://maven.seqera.io/snapshots/"
                 url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
-                credentials(PasswordCredentials) {
-                    username nexusUsername
-                    password nexusPassword
+                credentials(AwsCredentials) {
+                    accessKey nexusUsername
+                    secretKey nexusPassword
                 }
             }
         }
@@ -393,17 +392,6 @@ task upload {
     dependsOn coreProjects.publish
 }
 
-/*
- * Configure Nextflow staging plugin -- https://github.com/Codearte/gradle-nexus-staging-plugin
- * It adds the tasks
- * - closeRepository
- * - releaseRepository
- * - closeAndReleaseRepository
- */
-nexusStaging {
-    packageGroup = 'io.nextflow'
-    delayBetweenRetriesInMillis = 10_000
-}
 
 if( System.env.BUILD_PACK ) {
     apply from: 'packing.gradle'

build.gradle

@@ -12,7 +12,7 @@ group = "io.nextflow"
 
 dependencies {
     implementation ('com.amazonaws:aws-java-sdk-s3:1.12.777')
-    implementation 'com.google.code.gson:gson:2.10.1'
+    implementation 'com.google.code.gson:gson:2.13.1'
 }
 
 gradlePlugin {

buildSrc/build.gradle

@@ -1,5 +1,164 @@
 NEXTFLOW CHANGE-LOG
 ===================
+25.06.-0-edge - 6 Jul 2025
+- Add Git response max length check (#6190) [098fe84f]
+- Add docker.registryOverride option (#6205) [e26abe4f]
+- Add namespaces to nf-lang (#6176) [c839799b]
+- Add support for Bitbucket API token (#6209) [f711f24e]
+- Fix AWS NIO tests [069653dd]
+- Fix JWT token refresh to Fusion validation request (#6231) [95dfdafd]
+- Fix Apptainer and singularity --no-https option (#6216) [66ead2c6]
+- Fix class not found exception Google Life science executor (#6193) [7bfb9358]
+- Fix false error on nested closure in config process directive (#6203) [481c773b]
+- Fix false error when using stdin/stdout in a tuple (#6219) [5d7f12ce]
+- Fix false warning in config schema validator (#6240) [4e55ceeb]
+- Fix http response err message [1f05451f]
+- Fix mamba env create stalls at prompt [e7c93791]
+- Fix null reference error in lint command (#6202) [3e1837e6]
+- Fix readBytes for non-GitHub providers (#6243) [fa79c12c]
+- Fix replace List.getFirst with List.get(0) for compatibility [83428ee2]
+- Fix same Conda env is re-created for same module (#6166) [b46f01f9]
+- Fix stack overflow error with workflow metadata (#5988) [9d50c495]
+- Fix Maven publish with aws credentials [34d64ab3]
+- Implement retry strategy for Git repository provider (#6195) [0f922826]
+- Implement unordered directory hash (#6197) [a0eeed8d]
+- Minor Azure Batch disk slot calculation demoted to debug (#6234) [c65955ce]
+- Remove deprecated GsonHelper class [700faddb]
+- Remove deprecated nexus-staging plugin [68b449bb]
+- Rename IncludeModuleNode to IncludeEntryNode (#6217) [1f456d9c]
+- Replace URLConnection with HttpClient (#6188) [d1e6836e]
+- Return null result when repository provider has not auth [9d7fd9fc]
+- Unwrap Failsafe exception cause in Wave client [0cb39df5]
+- Update Google dependencies (#6229) [8bd42acb]
+- Update stalebot (#6228) [27174df3]
+- Update Wave retryPolicy default values (#6222) [b8069a58]
+- Upgrade AWS instructions to AL2023 (#6183) [ed407095]
+- Upgrade to AWS Java SDK v2 (#6165) [fc99b447]
+- Upload Google Batch log on task exit (#6226) [78d9f473]
+- Bump Google instance type for testing [bde9ef1b]
+- Bump Logback version 1.5.18 [e3d5cedd]
+- Bump Slf4j version 2.0.17 [93199e09]
+- Bump gson version 2.13.1 [ab8e36a2]
+- Bump nf-wave@1.14.0 [71ecf264]
+- Bump nf-tower@1.13.0 [53d1bfb4]
+- Bump nf-google@1.22.1 [cdf56d14]
+- Bump nf-azure@1.18.0 [fc514062]
+- Bump nf-amazon@3.0.0 [ffe699bd]
+
+25.04.4 - 16 Jun 2024
+- Fix bug in generated Groovy code (#6082) [ead2f320]
+- Fix default imports in included configs (#6096) [831a577b]
+- Fix variable checking in v2 config parser (#6097) [686ff608]
+- Sort linter errors/warnings by source location (#6098) [f51f339c]
+
+24.10.8 - 6 Jun 2025
+- Add Fusion license validation [02b4dd74]
+- Bump nf-wave@1.7.5-patch1 [cf6af9c5]
+- Bump nf-tower@1.9.3-patch1 [bdef2177]
+
+24.04.5 - 6 Jun 2025
+- Add Fusion license validation [cb7210b9]
+- Bump nf-wave@1.4.2-patch2 [69b63342]
+- Bump nf-tower@1.9.1-patch1 [8bfdc673]
+
+25.05.0-edge - 2 Jun 2025
+- Add Failsafe retry mechanism in K8s (#6083) [9e675c6a]
+- Add Platform info to Fusion license (#6142) [75f1bc52]
+- Add support for Azure Managed identities on Azure worker nodes with Fusion (#6118) [37981a5f]
+- Add support for container build compression (#5964) [daeefa0c]
+- Deprecated condaFile attribute [9e52b2ad]
+- Fix bug in generated Groovy code (#6082) [f0620656]
+- Fix default imports in included configs (#6096) [8f790615]
+- Fix variable checking in v2 config parser (#6097) [c3e9367b]
+- Force overwritting to trace file (#6105) [b9dace93]
+- Improve documentation of splitCsv operator (#6131) [f09ae8d7]
+- Make RepositoryProvider readBytes public (#6138) [9221b112]
+- Remove Az Fusion environment variable (#6143) [6c595eac]
+- Remove deprecated spack options [4e5d32a3]
+- Remove task attempt from hash list and add note (#6123) [9b48df4c]
+- Return streams in Lineage search and subkeys methods (#6067) [e4af2308]
+- Sort linter errors/warnings by source location (#6098) [7c52fafb]
+- Sunset the Google LS executor (#6070) [06e0d426]
+- Bump Groovy to version 4.0.27 (#6125) [258e1790]
+- Bump jgit:7.1.1.202505221757-r [16b88aca]
+- Bump nf-wave@1.13.0 [ab1d2dfe]
+- Bump nf-tower@1.12.0 [d4d97dfc]
+- Bump nf-k8s@1.1.0 [14eb7892]
+- Bump nf-google@1.22.0 [0f6498f1]
+- Bump nf-azure@1.17.0 [0b491840]
+
+25.04.3 - 2 Jun 2025
+- Add Platform info to Fusion license (#6142) [375db65a]
+- Force overwritting to trace file (#6105) [59e9d88d]
+- Bump nf-tower@1.11.3 [f7509bce] 
+
+25.04.2 - 13 May 2025
+- Add check subcommand in lineage (#6074) [5ba67bca]
+- Fix issues with `lint` console output (#6064) [7405f513]
+- Remove obsolete conda create --mkdir option (#6010) [9fc44f76]
+
+25.04.1 - 12 May 2025
+- Add Beta annotation to LinStore [3bd0d731]
+- Add data lineage guide (#6044) [7f7fb631]
+- Add more excludes to cmd lint [fc52c843]
+- Add support for ARM and Snapshots (#6057) [ef4c3865]
+- Add version to Lineage data model  (#6037) [5c2c2834]
+- Fix IllegalArgumentException on publish operator [ecdbe21b]
+- Fix linting of output `label` directive (#6065) [74bf33a2]
+- Fix selector expression documentation (#6050) [548c87bd]
+- Improve lineage checksum validation [d708bf09]
+- Improve not found error in lineage `render` (#6061) [e55419ad]
+- Make config parser v2 compatible with AssetManager (#6042) [aab5574b]
+- Refactor lineage model using v1beta1 package name [63d59154]
+- Support fromPath with pattern in Lineage Pseudo-FS (#6054) [ce165c0e]
+- Bump plugin template to v0.2.0 (#6062) [96c88931]
+- Bump nf-wave@1.12.1 [8cabfd74]
+
+25.04.0 - 8 May 2025
+- Add Support for Google Batch network tags (#5951) [d6e4d6c2]
+- Add Trace observer v2 (#6014) [c4437c1a]
+- Add allowed plugins list (#6049) [e22882e3]
+- Add azure.batch.jobMaxWallClockTime config option (#5996) [74963fdc]
+- Add config error checking to `run` command (#5998) [b0191fa4]
+- Add lineage to gitignore [02e1a542]
+- Add plugin create command (#6032) [68865f7f]
+- Add socketTimeoutException to retryable condition [b676e996]
+- Add support for Java 24 [a47ceefe]
+- Add timezone to lineage list timestamp [77736b55]
+- Add verbose AWS Batch job cleanup logging [504bd2df]
+- Add Support downloading plugins from OCI registries (#6016) [b41c65d5]
+- Add Support prefetching plugins metadata from plugins registry (#5887) [37d433d8]
+- Bring topic channels out of preview (#6030) [4b1b008b]
+- Data Lineage quick fixes (#6045) [f3cee76b]
+- Data Lineage workflow outputs fixes (#6041) [2cb49d08]
+- Data Lineage documentation (#5999) [c0bfc092]
+- Data Lineage programmatic API (#6003) [85b9d00d]
+- Fix NPE when params value is null [c7316eaf]
+- Fix `lineage find` command arguments validation (#6034) [375dc17c]
+- Fix concurrent modification error in `lint` command (#6053) [ccf5705e]
+- Fix errors in output fragment (#6020) [f947c9b4]
+- Fix false error in workflow output (#5982) [039ad401]
+- Fix small formatting inconsistencies  (#6024) [c3a47605]
+- Fix task progress percentage with failed tasks (#5868) [7e931e83]
+- Fix timeout in failing test [cf3e1661]
+- Improve plugin template update [d33c5063]
+- Improve process creation logging [0121670c]
+- Make plugins log less verbose [38410fd0]
+- Minor allow null output params and values [53824bb0]
+- Refactor lineage annotations as list (#6022) [9280b59e]
+- Remove test constructors or mark as TestOnly (#5216) [d4fadd42]
+- Replace output `labels` with `label` directive (#6035) [f99bcd33]
+- Separate language and runtime reference docs (#6018) [75ad21d0]
+- Split standard library docs into multiple pages (#5993) [2f82d236]
+- Store lineage meta in root location and default to .lineage path [d1cae3be]
+- Use `channel` namespace to access channel factories (#6029) [26557f0c]
+- Use the same antlr4 dep as Groovy 4 (#5990) [bd33a76e]
+- Bump plugin template v0.1.0 [b54e78f3]
+- Bump repository provider connection timeout to 60 secs [39aaa831]
+- Bump nf-google@1.21.0 [b7aba74d]
+- Bump nf-azure@1.16.0 [e8304c1e]
+- Bump nf-amazon@2.15.0 [c5100f6d]
+
 24.10.6 - 24 Apr 2025
 - Add support for Fusion Snapshots (#5954) [76a997a3]
 - Bump nf-wave@1.7.5 [3ddf16fb]

changelog.txt

@@ -46,8 +46,10 @@ If the access credentials are not found in the above file, Nextflow looks for AW
 1. The `nextflow.config` file in the pipeline execution directory
 2. The environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
 3. The environment variables `AWS_ACCESS_KEY` and `AWS_SECRET_KEY`
-4. The `default` profile in the AWS credentials file located at `~/.aws/credentials`
-5. The `default` profile in the AWS client configuration file located at `~/.aws/config`
+4. The profile in the AWS credentials file located at `~/.aws/credentials`
+    - Uses the `default `profile or the environment variable `AWS_PROFILE` if set
+5. The profile in the AWS client configuration file located at `~/.aws/config`
+    - Uses the `default `profile or the environment variable `AWS_PROFILE` if set
 6. The temporary AWS credentials provided by an IAM instance role. See [IAM Roles](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) documentation for details.
 
 More information regarding [AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html) are available in the AWS documentation.

docs/amazons3.md

@@ -86,6 +86,8 @@ Minimal permissions policies to be attached to the AWS account used by Nextflow
   "ecr:ListTagsForResource"
   ```
 
+  Alternatively, you can use AWS provided `AmazonEC2ContainerRegistryReadOnly` managed policy.
+
 :::{note}
 If you are running Fargate or Fargate Spot, you may need the following policies in addition to the listed above:
   ```json
@@ -121,7 +123,9 @@ Depending on the pipeline configuration, the above actions can be done all in a
                 "Action": [
                     "s3:GetObject",
                     "s3:PutObject",
-                    "s3:DeleteObject"
+                    "s3:DeleteObject",
+                    "s3:PutObjectTagging",
+                    "s3:AbortMultipartUpload"
                 ],
                 "Resource": "arn:aws:s3:::<bucket name>/*"
             },
@@ -277,7 +281,7 @@ There are several reasons why you might need to create your own [AMI (Amazon Mac
 ### Create your custom AMI
 
 From the EC2 Dashboard, select **Launch Instance**, then select **Browse more AMIs**. In the new page, select
-**AWS Marketplace AMIs**, and then search for **Amazon ECS-Optimized Amazon Linux 2 (AL2) x86_64 AMI**. Select the AMI and continue as usual to configure and launch the instance.
+**AWS Marketplace AMIs**, and then search for `Amazon ECS-Optimized Amazon Linux 2 (AL2) x86_64 AMI`. Select the AMI and continue as usual to configure and launch the instance.
 
 :::{note}
 The selected instance has a root volume of 30GB. Make sure to increase its size or add a second EBS volume with enough storage for real genomic workloads.
@@ -346,14 +350,13 @@ The grandparent directory of the `aws` tool will be mounted into the container a
 
 ### Docker installation
 
-Docker is required by Nextflow to execute tasks on AWS Batch. The **Amazon ECS-Optimized Amazon Linux 2 (AL2) x86_64 AMI** has Docker installed, however, if you create your AMI from a different AMI that does not have Docker installed, you will need to install it manually.
+Docker is required by Nextflow to execute tasks on AWS Batch. The **Amazon ECS-optimized Amazon Linux 2023 AMI** has Docker installed, however, if you create your AMI from a different AMI that does not have Docker installed, you will need to install it manually.
 
 The following snippet shows how to install Docker on an Amazon EC2 instance:
 
 ```bash
 # install Docker
 sudo yum update -y
-sudo amazon-linux-extras install docker
 sudo yum install docker
 
 # start the Docker service
@@ -363,21 +366,20 @@ sudo service docker start
 sudo usermod -a -G docker ec2-user
 ```
 
-You may have to reboot your instance for the changes to `ec2-user` to take effect.
+You must logging out and logging back in again to use the new `ec2-user` permissions.
 
 These steps must be done *before* creating the AMI from the current EC2 instance.
 
 ### Amazon ECS container agent installation
 
 The [ECS container agent](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_agent.html) is a component of Amazon Elastic Container Service (Amazon ECS) and is responsible for managing containers on behalf of ECS. AWS Batch uses ECS to execute containerized jobs, therefore it requires the agent to be installed on EC2 instances within your Compute Environments.
 
-The ECS agent is included in the **Amazon ECS-Optimized Amazon Linux 2 (AL2) x86_64 AMI** . If you use a different base AMI, you can also install the agent on any EC2 instance that supports the Amazon ECS specification.
+The ECS agent is included in the **Amazon ECS-optimized Amazon Linux 2023 AMI** . If you use a different base AMI, you can also install the agent on any EC2 instance that supports the Amazon ECS specification.
 
 To install the agent, follow these steps:
 
 ```bash
-sudo amazon-linux-extras disable docker
-sudo amazon-linux-extras install -y ecs
+sudo yum install ecs-init
 sudo systemctl enable --now ecs
 ```