diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 74be2fdd20..7fd6f5cded 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -915,7 +915,8 @@ The `lint` command parses and analyzes the given Nextflow scripts and config fil **Options** `-exclude` -: File pattern to exclude from linting. Can be specified multiple times (default: `.git, .nf-test, work`). +: File pattern to exclude from linting (default: `.git, .lineage, .nextflow, .nf-test, nf-test.config, work`). +: Can be specified multiple times. `-format` : Format scripts and config files that have no errors. @@ -923,6 +924,11 @@ The `lint` command parses and analyzes the given Nextflow scripts and config fil `-o, -output` : Output mode for reporting errors: `full`, `extended`, `concise`, `json`, `markdown` (default: `full`). +`-project-dir` +: :::{versionadded} 26.04.0 + ::: +: Path to project directory (default: `'.'`). Used to locate project-level assets such as the lib directory and modules directory. + `-sort-declarations` : Sort script declarations in Nextflow scripts (default: `false`). diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 3a33bbc8a6..4c987c8f4b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -80,7 +80,7 @@ import nextflow.trace.event.FilePublishEvent import nextflow.trace.event.TaskEvent import nextflow.trace.event.WorkflowOutputEvent import nextflow.util.Barrier -import nextflow.util.ConfigHelper +import nextflow.util.ClassLoaderFactory import nextflow.util.Duration import nextflow.util.HistoryFile import nextflow.util.LoggerHelper @@ -602,21 +602,8 @@ class Session implements ISession { ScriptBinding getBinding() { binding } @Memoized - ClassLoader getClassLoader() { getClassLoader0() } - - @PackageScope - ClassLoader getClassLoader0() { - // extend the class-loader if required - final gcl = new GroovyClassLoader() - final libraries = ConfigHelper.resolveClassPaths(getLibDir()) - - for( Path lib : libraries ) { - def path = lib.complete() - log.debug "Adding to the classpath library: ${path}" - gcl.addClasspath(path.toString()) - } - - return gcl + ClassLoader getClassLoader() { + ClassLoaderFactory.create(getLibDir()) } Barrier getBarrier() { monitorsBarrier } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLint.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdLint.groovy index 4b180e0508..556ee0d542 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLint.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdLint.groovy @@ -40,6 +40,7 @@ import nextflow.script.formatter.ScriptFormattingVisitor import nextflow.script.parser.v2.ErrorListener import nextflow.script.parser.v2.ErrorSummary import nextflow.script.parser.v2.StandardErrorListener +import nextflow.util.ClassLoaderFactory import nextflow.util.PathUtils import org.codehaus.groovy.control.SourceUnit import org.codehaus.groovy.control.messages.SyntaxErrorMessage @@ -62,7 +63,7 @@ class CmdLint extends CmdBase { names = ['-exclude'], description = 'File pattern to exclude from error checking (can be specified multiple times)' ) - List excludePatterns = ['.git', '.lineage', '.nf-test', '.nextflow', 'work', 'nf-test.config'] + List excludePatterns = ['.git', '.lineage', '.nextflow', '.nf-test', 'nf-test.config', 'work'] @Parameter( names = ['-o', '-output'], @@ -82,6 +83,12 @@ class CmdLint extends CmdBase { } } + @Parameter( + names = ['-project-dir'], + description = 'Path to project directory (default: .)' + ) + String projectDir = '.' + @Parameter(names = ['-format'], description = 'Format scripts and config files that have no errors') boolean formatting @@ -121,7 +128,11 @@ class CmdLint extends CmdBase { if( !spaces && !tabs ) spaces = 4 - scriptParser = new ScriptParser() + final baseDir = Path.of(projectDir) + final libDir = baseDir.resolve('lib') + final classLoader = ClassLoaderFactory.create([ libDir ]) + + scriptParser = new ScriptParser(baseDir, classLoader) configParser = new ConfigParser() errorListener = outputMode == 'json' ? new JsonErrorListener() diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ClassLoaderFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ClassLoaderFactory.groovy new file mode 100644 index 0000000000..84cd64bc6d --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/util/ClassLoaderFactory.groovy @@ -0,0 +1,78 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.util + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +/** + * Helper methods to create class loaders with + * additional classpaths. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ClassLoaderFactory { + + /** + * Create a class loader with the given directories + * added to the classpath. + * + * @param dirs + */ + static GroovyClassLoader create(List dirs) { + final gcl = new GroovyClassLoader() + final libraries = resolveClassPaths(dirs) + for( final lib : libraries ) { + gcl.addClasspath(lib.complete().toString()) + } + return gcl + } + + /** + * Given a list of directories, look for the files ending with + * the '.jar' extension and return a list containing the original + * directories and the JAR paths. + * + * @param dirs + */ + static List resolveClassPaths(List dirs) { + + List result = [] + + if( !dirs ) + return result + + for( final path : dirs ) { + if( path.isFile() && path.name.endsWith('.jar') ) { + result << path + } + else if( path.isDirectory() ) { + result << path + path.eachFileMatch( ~/.+\.jar$/ ) { + if( it.isFile() ) + result << it + } + } + } + + return result + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy index 8b4365e2fb..dd6f11d296 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy @@ -16,8 +16,6 @@ package nextflow.util -import java.nio.file.Path - import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.transform.PackageScope @@ -69,32 +67,6 @@ class ConfigHelper { return obj } - /** - * Given a list of paths looks for the files ending with the extension '.jar' and return - * a list containing the original directories, plus the JARs paths - * - * @param dirs - * @return - */ - static List resolveClassPaths( List dirs ) { - - List result = [] - if( !dirs ) - return result - - for( Path path : dirs ) { - if( path.isFile() && path.name.endsWith('.jar') ) { - result << path - } - else if( path.isDirectory() ) { - result << path - path.eachFileMatch( ~/.+\.jar$/ ) { if(it.isFile()) result << it } - } - } - - return result - } - static private final String TAB = ' ' static private void canonicalFormat(StringBuilder writer, ConfigObject object, int level, boolean sort, List stack) { diff --git a/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy b/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy index 1b0aff867b..4cd4fcccd6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy @@ -54,7 +54,7 @@ class RemoteSession implements Serializable, Closeable { private transient List localPaths = deserialiseClasspath() @Lazy - private transient List resolvedClasspath = ConfigHelper.resolveClassPaths(localPaths) + private transient List resolvedClasspath = ClassLoaderFactory.resolveClassPaths(localPaths) protected RemoteSession() { } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLintTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdLintTest.groovy index f62b58e113..eb21ce8656 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLintTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdLintTest.groovy @@ -81,4 +81,39 @@ class CmdLintTest extends Specification { dir?.deleteDir() } + def 'should add lib directory to class loader' () { + + given: + def dir = Files.createTempDirectory('test') + + dir.resolve('main.nf').text = '''\ + println Utils.hello() + ''' + + dir.resolve('lib').mkdir() + dir.resolve('lib/Utils.groovy').text = '''\ + class Utils { + + String hello() { + return 'Hello!' + } + } + ''' + + when: + def cmd = new CmdLint() + cmd.args = [dir.toString()] + cmd.projectDir = dir.toString() + cmd.launcher = Mock(Launcher) { + getOptions() >> Mock(CliOptions) + } + cmd.run() + + then: + noExceptionThrown() + + cleanup: + dir?.deleteDir() + } + } diff --git a/modules/nextflow/src/test/groovy/nextflow/util/ClassLoaderFactoryTest.groovy b/modules/nextflow/src/test/groovy/nextflow/util/ClassLoaderFactoryTest.groovy new file mode 100644 index 0000000000..2057252a57 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/util/ClassLoaderFactoryTest.groovy @@ -0,0 +1,59 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.util + +import java.nio.file.Files +import java.nio.file.Path + +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class ClassLoaderFactoryTest extends Specification { + + def testResolveClasspaths() { + + given: + def path1 = Files.createTempDirectory('path1') + path1.resolve('file1').text = 'File 1' + path1.resolve('file2.jar').text = 'File 2' + path1.resolve('dir').mkdir() + path1.resolve('dir/file3').text = 'File 3' + path1.resolve('dir/file4').text = 'File 4' + + def path2 = Files.createTempDirectory('path2') + path2.resolve('file5').text = 'File 5' + path2.resolve('file6.jar').text = 'File 6' + + def path3 = Path.of('/some/file') + + when: + def list = ClassLoaderFactory.resolveClassPaths([path1, path2, path3]) + then: + list.size() == 4 + list.contains( path1 ) + list.contains( path1.resolve('file2.jar') ) + list.contains( path2 ) + list.contains( path2.resolve('file6.jar') ) + + cleanup: + path1?.deleteDir() + path2?.deleteDir() + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy index 40edd2a38c..b4968a6aae 100644 --- a/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy @@ -16,9 +16,6 @@ package nextflow.util -import java.nio.file.Files -import java.nio.file.Paths - import nextflow.config.ConfigClosurePlaceholder import spock.lang.Specification import spock.lang.Unroll @@ -47,37 +44,6 @@ class ConfigHelperTest extends Specification { } - def testResolveClasspaths() { - - given: - def path1 = Files.createTempDirectory('path1') - path1.resolve('file1').text = 'File 1' - path1.resolve('file2.jar').text = 'File 2' - path1.resolve('dir').mkdir() - path1.resolve('dir/file3').text = 'File 3' - path1.resolve('dir/file4').text = 'File 4' - - def path2 = Files.createTempDirectory('path2') - path2.resolve('file5').text = 'File 5' - path2.resolve('file6.jar').text = 'File 6' - - def path3 = Paths.get('/some/file') - - when: - def list = ConfigHelper.resolveClassPaths([path1, path2, path3]) - then: - list.size() == 4 - list.contains( path1 ) - list.contains( path1.resolve('file2.jar') ) - list.contains( path2 ) - list.contains( path2.resolve('file6.jar') ) - - cleanup: - path1?.deleteDir() - path2?.deleteDir() - - } - def 'should render config using properties notation' () { given: diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/GroovyCompiler.java b/modules/nf-lang/src/main/java/nextflow/script/control/GroovyCompiler.java new file mode 100644 index 0000000000..7067fe7564 --- /dev/null +++ b/modules/nf-lang/src/main/java/nextflow/script/control/GroovyCompiler.java @@ -0,0 +1,65 @@ +/* + * Copyright 2024-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nextflow.script.control; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +import groovy.lang.GroovyClassLoader; +import org.codehaus.groovy.GroovyBugError; +import org.codehaus.groovy.ast.ClassNode; +import org.codehaus.groovy.control.CompilationUnit; +import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.Phases; +import org.codehaus.groovy.control.SourceUnit; + +/** + * Load Groovy classes from the `lib` directory. + * + * @author Ben Sherman + */ +public class GroovyCompiler { + + public static List compile(SourceUnit su) { + // create compilation unit + var config = new CompilerConfiguration(); + config.getOptimizationOptions().put(CompilerConfiguration.GROOVYDOC, true); + var classLoader = new GroovyClassLoader(); + var compilationUnit = new CompilationUnit(config, null, classLoader); + + // create source units (or restore from cache) + var uri = su.getSource().getURI(); + var sourceUnit = new SourceUnit( + new File(uri), + config, + classLoader, + new LazyErrorCollector(config)); + compilationUnit.addSource(sourceUnit); + + // compile source files + compilationUnit.compile(Phases.CANONICALIZATION); + + // collect compiled classes + var moduleNode = sourceUnit.getAST(); + if( moduleNode == null ) + return Collections.emptyList(); + + return moduleNode.getClasses(); + } + +} diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java index f3c421c672..3d31e70b8e 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java @@ -264,9 +264,16 @@ protected ClassNode resolveFromClassResolver(String name) { var lookupResult = classNodeResolver.resolveName(name, compilationUnit); if( lookupResult == null ) return null; - if( !lookupResult.isClassNode() ) - throw new GroovyBugError("class resolver lookup result is not a class node"); - return lookupResult.getClassNode(); + if( lookupResult.isClassNode() ) + return lookupResult.getClassNode(); + // When a Groovy class from the lib directory is used, the class + // loader returns the URI of the Groovy file. We only need to compile + // the Groovy file enough to resolve the class definition for the purpose + // of name checking. + var su = lookupResult.getSourceUnit(); + return GroovyCompiler.compile(su).stream() + .filter(cn -> cn.getName().equals(name)) + .findFirst().orElse(null); } /** diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ScriptParser.java b/modules/nf-lang/src/main/java/nextflow/script/control/ScriptParser.java index 62ba17a7a1..ea70bdb2b4 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ScriptParser.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ScriptParser.java @@ -37,11 +37,13 @@ public class ScriptParser { private Path projectDir; private Compiler compiler; - public ScriptParser(Path projectDir) { + public ScriptParser(Path projectDir, GroovyClassLoader classLoader) { this.projectDir = projectDir; - var config = getConfig(); - var classLoader = new GroovyClassLoader(); - this.compiler = new Compiler(config, classLoader); + this.compiler = new Compiler(getConfig(), classLoader); + } + + public ScriptParser(Path projectDir) { + this(projectDir, new GroovyClassLoader()); } public ScriptParser() { diff --git a/modules/nf-lang/src/test/groovy/nextflow/script/control/ResolveIncludeTest.groovy b/modules/nf-lang/src/test/groovy/nextflow/script/control/ResolveIncludeTest.groovy index be60551f23..e4f9939892 100644 --- a/modules/nf-lang/src/test/groovy/nextflow/script/control/ResolveIncludeTest.groovy +++ b/modules/nf-lang/src/test/groovy/nextflow/script/control/ResolveIncludeTest.groovy @@ -33,8 +33,8 @@ import static test.TestUtils.tempFile */ class ResolveIncludeTest extends Specification { - List check(List files) { - final parser = new ScriptParser() + List check(Path projectDir, List files) { + final parser = new ScriptParser(projectDir) return TestUtils.check(parser, files) } @@ -48,7 +48,7 @@ class ResolveIncludeTest extends Specification { def module = root.resolve('hello.nf') when: - def errors = check([main]) + def errors = check(root, [main]) then: errors.size() == 1 errors[0].getStartLine() == 1 @@ -72,7 +72,7 @@ class ResolveIncludeTest extends Specification { ''') when: - def errors = check([main, module]) + def errors = check(root, [main, module]) then: errors.size() == 2 errors[0].getSourceLocator().endsWith('main.nf') @@ -102,7 +102,7 @@ class ResolveIncludeTest extends Specification { ''') when: - def errors = check([main, module]) + def errors = check(root, [main, module]) then: errors.size() == 1 errors[0].getSourceLocator().endsWith('main.nf') @@ -128,7 +128,7 @@ class ResolveIncludeTest extends Specification { ''') when: - def errors = check([main, module]) + def errors = check(root, [main, module]) then: errors.size() == 0