Merge branch 'master' into 3129-fix-workflow-complete-handler

Author: jorgee

modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy

@@ -1946,7 +1946,7 @@ class TaskProcessor {
             if( item instanceof Path || coerceToPath ) {
                 final path = resolvePath(item)
                 final target = executor.isForeignFile(path) ? foreignFiles.addToForeign(path) : path
-                final holder = new FileHolder(target)
+                final holder = new FileHolder(path, target)
                 files << holder
             }
             else {
@@ -2211,6 +2211,13 @@ class TaskProcessor {
             keys.add( it.value )
         }
 
+        // add all eval commands (they are part of the script)
+        for( Map.Entry<OutParam,Object> it : task.outputs ) {
+            if( it.key instanceof CmdEvalParam ) {
+                keys.add(((CmdEvalParam) it.key).getTarget(task.context))
+            }
+        }
+
         // add all variable references in the task script but not declared as input/output
         def vars = getTaskGlobalVars(task)
         if( vars ) {
@@ -2245,6 +2252,7 @@ class TaskProcessor {
             }
         }
 
+        // add stub-run flag
         if( session.stubRun && task.config.getStubBlock() ) {
             keys.add('stub-run')
         }

modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy

@@ -41,6 +41,7 @@ import nextflow.script.BodyDef
 import nextflow.script.ProcessConfig
 import nextflow.script.ScriptType
 import nextflow.script.bundle.ResourcesBundle
+import nextflow.script.params.CmdEvalParam
 import nextflow.script.params.FileInParam
 import nextflow.script.params.FileOutParam
 import nextflow.util.ArrayBag
@@ -1218,4 +1219,52 @@ class TaskProcessorTest extends Specification {
         0 * collector.collect(task)
         1 * exec.submit(task)
     }
+
+    def 'should use eval outputs in hash' () {
+        given:
+        def session = Mock(Session) {
+            getDumpHashes() >> 'json'
+            getUniqueId() >> UUID.fromString('b69b6eeb-b332-4d2c-9957-c291b15f498c')
+            getBinEntries() >> ['foo.sh': Paths.get('/some/path/foo.sh'), 'bar.sh': Paths.get('/some/path/bar.sh')]
+        }
+        def outParam = Mock(CmdEvalParam) {
+            getTarget( _ as Map) >> 'foo --version'
+        }
+        def task1 = Spy(TaskRun) {
+            getSource() >> 'hello world'
+            lazyName() >> 'hello'
+            isContainerEnabled() >> false
+            getContainer() >> null
+            getSpackEnv() >> null
+            getCondaEnv() >> null
+            getConfig() >> Mock(TaskConfig)
+            getContext() >> [:]
+        }
+        def task2 = Spy(TaskRun) {
+            getSource() >> 'hello world'
+            lazyName() >> 'hello'
+            isContainerEnabled() >> false
+            getContainer() >> null
+            getSpackEnv() >> null
+            getCondaEnv() >> null
+            getConfig() >> Mock(TaskConfig)
+            getContext() >> [:]
+        }
+        task2.setOutput(outParam)
+
+        def processor = Spy(TaskProcessor){
+            getTaskGlobalVars( _ as TaskRun) >> [foo:'a', bar:'b']
+        }
+        processor.@session = session
+        processor.@config = Mock(ProcessConfig)
+
+        when:
+        def uuid1 = processor.createTaskHashKey(task1)
+        def uuid2 = processor.createTaskHashKey(task2)
+
+        then:
+        uuid1 != uuid2
+
+    }
+
 }

modules/nf-commons/src/main/nextflow/file/FileHolder.groovy

@@ -20,19 +20,23 @@ package nextflow.file
 import java.nio.file.FileSystems
 import java.nio.file.Path
 
+import com.google.common.hash.Hasher
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
 import groovy.transform.PackageScope
 import groovy.transform.ToString
 import groovy.util.logging.Slf4j
+import nextflow.util.CacheFunnel
+import nextflow.util.CacheHelper
+import nextflow.util.CacheHelper.HashMode
 /**
  * Implements a special {@code Path} used to stage files in the work area
  */
 @Slf4j
 @ToString(includePackage = false, includeNames = true)
 @EqualsAndHashCode
 @CompileStatic
-class FileHolder  {
+class FileHolder implements CacheFunnel {
 
     final def sourceObj
 
@@ -52,7 +56,7 @@ class FileHolder  {
         assert path != null
 
         this.sourceObj = origin
-        this.storePath = path
+        this.storePath = real(path)
         this.stageName = norm(path.getFileName())
     }
 
@@ -66,10 +70,19 @@ class FileHolder  {
         new FileHolder( this.sourceObj, this.storePath, stageName )
     }
 
+    Path getSourcePath() {
+        sourceObj instanceof Path ? sourceObj : null
+    }
+
     Path getStorePath() { storePath }
 
     String getStageName() { stageName }
 
+    @Override
+    Hasher funnel(Hasher hasher, HashMode mode) {
+        return CacheHelper.hasher(hasher, sourceObj, mode)
+    }
+
     @PackageScope
     static FileHolder get( def path, def name = null ) {
         Path storePath = path as Path

modules/nf-commons/src/main/nextflow/util/HashBuilder.java

@@ -44,7 +44,6 @@
 import nextflow.ISession;
 import nextflow.extension.Bolts;
 import nextflow.extension.FilesEx;
-import nextflow.file.FileHolder;
 import nextflow.io.SerializableMarker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -162,9 +161,6 @@ else if( value instanceof Collection)
             for( Object item : ((Collection)value) )
                 with(item);
 
-        else if( value instanceof FileHolder )
-            with(((FileHolder) value).getSourceObj());
-
         else if( value instanceof Path )
             hashFile(hasher, (Path)value, mode, basePath);