diff --git a/docs/strict-syntax.md b/docs/strict-syntax.md index ee1d9b62d8..284a523f8c 100644 --- a/docs/strict-syntax.md +++ b/docs/strict-syntax.md @@ -534,7 +534,7 @@ To enable these checks, set **Nextflow > Error reporting mode** to **paranoid** ### Using params outside the entry workflow -While params can be used anywhere in the pipeline code, they are only intended to be used in the entry workflow. +While params can be used anywhere in the pipeline code, they are only intended to be used in the entry workflow and the `output` block. As a best practice, processes and workflows should receive params as explicit inputs: diff --git a/docs/workflow.md b/docs/workflow.md index 1574b974fe..973591ad55 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -49,13 +49,7 @@ params { } ``` -The following types can be used for parameters: - -- {ref}`stdlib-types-boolean` -- {ref}`stdlib-types-float` -- {ref}`stdlib-types-integer` -- {ref}`stdlib-types-path` -- {ref}`stdlib-types-string` +All {ref}`standard types ` except for the dataflow types (`Channel` and `Value`) can be used for parameters. Parameters can be used in the entry workflow: @@ -66,12 +60,18 @@ workflow { ``` :::{note} -As a best practice, parameters should only be used directly in the entry workflow and passed to workflows and processes as explicit inputs. +As a best practice, parameters should only be referenced in the entry workflow or `output` block. Parameters can be passed to workflows and processes as explicit inputs. ::: The default value can be overridden by the command line, params file, or config file. Parameters from multiple sources are resolved in the order described in {ref}`cli-params`. Parameters specified on the command line are converted to the appropriate type based on the corresponding type annotation. -A parameter that doesn't specify a default value is a *required* param. If a required param is not given a value at runtime, the run will fail. +A parameter that doesn't specify a default value is a *required* parameter. If a required parameter is not given a value at runtime, the run will fail. + +Parameters with a collection type (i.e., `List`, `Set`, or `Bag`) can be supplied a file path instead of a literal collection. The file must be CSV, JSON, or YAML. Nextflow will parse the file contents and assign the resuling collection to the parameter. An error is thrown if the file contents do not match the parameter type. + +:::{note} +When supplying a CSV file to a collection parameter, the CSV file must contain a header row and must use a comma (`,`) as the column separator. +::: (workflow-params-legacy)= diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy index 0afb359aff..0f387b512a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy @@ -88,7 +88,7 @@ class CmdKubeRun extends CmdRun { @Override void run() { final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List - final pipeline = stdin ? '-' : ( args ? args[0] : null ) + final pipeline = args ? args[0] : null if( !pipeline ) throw new AbortOperationException("No project name was specified") if( hasAnsiLogFlag() ) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index bb6d641cc8..57550c393d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -156,9 +156,6 @@ class CmdRun extends CmdBase implements HubOptions { @Parameter(names=['-latest'], description = 'Pull latest changes before run') boolean latest - @Parameter(names='-stdin', hidden = true) - boolean stdin - @Parameter(names = ['-ansi'], hidden = true, arity = 0) void setAnsi(boolean value) { launcher.options.ansiLog = value @@ -289,18 +286,10 @@ class CmdRun extends CmdBase implements HubOptions { @Override String getName() { NAME } - String getParamsFile() { - return paramsFile ?: sysEnv.get('NXF_PARAMS_FILE') - } - - boolean hasParams() { - return params || getParamsFile() - } - @Override void run() { final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List - final pipeline = stdin ? '-' : ( args ? args[0] : null ) + final pipeline = args ? args[0] : null if( !pipeline ) throw new AbortOperationException("No project name was specified") @@ -598,20 +587,6 @@ class CmdRun extends CmdBase implements HubOptions { protected ScriptFile getScriptFile0(String pipelineName) { assert pipelineName - /* - * read from the stdin - */ - if( pipelineName == '-' ) { - def file = tryReadFromStdin() - if( !file ) - throw new AbortOperationException("Cannot access `stdin` stream") - - if( revision ) - throw new AbortOperationException("Revision option cannot be used when running a script from stdin") - - return new ScriptFile(file) - } - /* * look for a file with the specified pipeline name */ @@ -661,171 +636,9 @@ class CmdRun extends CmdBase implements HubOptions { } - static protected File tryReadFromStdin() { - if( !System.in.available() ) - return null - - getScriptFromStream(System.in) - } - - static protected File getScriptFromStream( InputStream input, String name = 'nextflow' ) { - input != null - File result = File.createTempFile(name, null) - result.deleteOnExit() - input.withReader { Reader reader -> result << reader } - return result - } - @Memoized // <-- avoid parse multiple times the same file and params Map parsedParams(Map configVars) { - - final result = [:] - - // apply params file - final file = getParamsFile() - if( file ) { - def path = validateParamsFile(file) - def type = path.extension.toLowerCase() ?: null - if( type == 'json' ) - readJsonFile(path, configVars, result) - else if( type == 'yml' || type == 'yaml' ) - readYamlFile(path, configVars, result) - } - - // apply CLI params - if( !params ) - return result - - for( Map.Entry entry : params ) { - addParam( result, entry.key, entry.value ) - } - return result - } - - - static final private Pattern DOT_ESCAPED = ~/\\\./ - static final private Pattern DOT_NOT_ESCAPED = ~/(?() - params.put(root, nested) - } - else if( nested !instanceof Map ) { - log.warn "Command line parameter --${path.join('.')} is overwritten by --${fullKey}" - nested = new LinkedHashMap<>() - params.put(root, nested) - } - addParam((Map)nested, key.substring(p+1), value, path, fullKey) - } - else { - addParam0(params, key.replaceAll(DOT_ESCAPED,'.'), parseParamValue(value)) - } - } - - static protected void addParam0(Map params, String key, Object value) { - if( key.contains('-') ) - key = kebabToCamelCase(key) - params.put(key, value) - } - - static protected String kebabToCamelCase(String str) { - final result = new StringBuilder() - str.split('-').eachWithIndex { String entry, int i -> - result << (i>0 ? StringUtils.capitalize(entry) : entry ) - } - return result.toString() - } - - static protected parseParamValue(String str) { - if ( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || NF.isSyntaxParserV2() ) - return str - - if ( str == null ) return null - - if ( str.toLowerCase() == 'true') return Boolean.TRUE - if ( str.toLowerCase() == 'false' ) return Boolean.FALSE - - if ( str==~/-?\d+(\.\d+)?/ && str.isInteger() ) return str.toInteger() - if ( str==~/-?\d+(\.\d+)?/ && str.isLong() ) return str.toLong() - if ( str==~/-?\d+(\.\d+)?/ && str.isDouble() ) return str.toDouble() - - return str - } - - private Path validateParamsFile(String file) { - - def result = FileHelper.asPath(file) - def ext = result.getExtension() - if( !VALID_PARAMS_FILE.contains(ext) ) - throw new AbortOperationException("Not a valid params file extension: $file -- It must be one of the following: ${VALID_PARAMS_FILE.join(',')}") - - return result - } - - static private Pattern PARAMS_VAR = ~/(?m)\$\{(\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)}/ - - protected String replaceVars0(String content, Map binding) { - content.replaceAll(PARAMS_VAR) { List matcher -> - // - the regex matcher is represented as list - // - the first element is the matching string ie. `${something}` - // - the second element is the group content ie. `something` - // - make sure the regex contains at least a group otherwise the closure - // parameter is a string instead of a list of the call fail - final placeholder = matcher.get(0) - final key = matcher.get(1) - - if( !binding.containsKey(key) ) { - final msg = "Missing params file variable: $placeholder" - if(NF.strictMode) - throw new AbortOperationException(msg) - log.warn msg - return placeholder - } - - return binding.get(key) - } - } - - private void readJsonFile(Path file, Map configVars, Map result) { - try { - def text = configVars ? replaceVars0(file.text, configVars) : file.text - def json = (Map) new JsonSlurper().parseText(text) - json.forEach((name, value) -> { - addParam0(result, name, value) - }) - } - catch (NoSuchFileException | FileNotFoundException e) { - throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}") - } - catch( Exception e ) { - throw new AbortOperationException("Cannot parse params file: ${file.toUriString()} - Cause: ${e.message}", e) - } - } - - private void readYamlFile(Path file, Map configVars, Map result) { - try { - def text = configVars ? replaceVars0(file.text, configVars) : file.text - def yaml = (Map) new Yaml().load(text) - yaml.forEach((name, value) -> { - addParam0(result, name, value) - }) - } - catch (NoSuchFileException | FileNotFoundException e) { - throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}") - } - catch( Exception e ) { - throw new AbortOperationException("Cannot parse params file: ${file.toUriString()}", e) - } + return new ParamsCollector(params, paramsFile).apply(configVars) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/ParamsCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/ParamsCollector.groovy new file mode 100644 index 0000000000..b164538e9a --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/ParamsCollector.groovy @@ -0,0 +1,201 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli + +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.util.regex.Pattern + +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.NF +import nextflow.SysEnv +import nextflow.exception.AbortOperationException +import nextflow.file.FileHelper +import org.apache.commons.lang3.StringUtils +import org.yaml.snakeyaml.Yaml +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ParamsCollector { + + private Map params + + private String paramsFile + + ParamsCollector(Map params, String paramsFile) { + this.params = params + this.paramsFile = paramsFile ?: SysEnv.get('NXF_PARAMS_FILE') + } + + Map apply(Map configVars) { + + final result = [:] + + // apply params from stdin + if( System.in.available() ) { + String text = System.in.text + if( configVars ) + text = replaceVars0(text, configVars) + final stdinParams = new JsonSlurper().parseText(text) as Map + stdinParams.forEach((name, value) -> { + addParam0(result, name, value) + }) + } + + // apply params file + if( paramsFile ) { + final path = validateParamsFile(paramsFile) + final type = path.extension.toLowerCase() ?: null + if( type == 'json' ) + readJsonFile(path, configVars, result) + else if( type == 'yml' || type == 'yaml' ) + readYamlFile(path, configVars, result) + } + + // apply CLI params + for( final entry : params ) + addParam(result, entry.key, entry.value) + + return result + } + + private static final Pattern DOT_ESCAPED = ~/\\\./ + private static final Pattern DOT_NOT_ESCAPED = ~/(?() + params.put(root, nested) + } + else if( nested !instanceof Map ) { + log.warn "Command line parameter --${path.join('.')} is overwritten by --${fullKey}" + nested = new LinkedHashMap<>() + params.put(root, nested) + } + addParam((Map)nested, key.substring(p+1), value, path, fullKey) + } + else { + addParam0(params, key.replaceAll(DOT_ESCAPED,'.'), parseParamValue(value)) + } + } + + protected static void addParam0(Map params, String key, Object value) { + if( key.contains('-') ) + key = kebabToCamelCase(key) + params.put(key, value) + } + + protected static String kebabToCamelCase(String str) { + final result = new StringBuilder() + str.split('-').eachWithIndex { String entry, int i -> + result << (i>0 ? StringUtils.capitalize(entry) : entry ) + } + return result.toString() + } + + protected static parseParamValue(String str) { + if( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || NF.isSyntaxParserV2() ) + return str + + if( str == null ) return null + + if( str.toLowerCase() == 'true') return Boolean.TRUE + if( str.toLowerCase() == 'false' ) return Boolean.FALSE + + if( str==~/-?\d+(\.\d+)?/ && str.isInteger() ) return str.toInteger() + if( str==~/-?\d+(\.\d+)?/ && str.isLong() ) return str.toLong() + if( str==~/-?\d+(\.\d+)?/ && str.isDouble() ) return str.toDouble() + + return str + } + + public static final List VALID_PARAMS_FILE = ['json', 'yml', 'yaml'] + + private Path validateParamsFile(String file) { + final result = FileHelper.asPath(file) + final ext = result.getExtension() + if( !VALID_PARAMS_FILE.contains(ext) ) + throw new AbortOperationException("Not a valid params file extension: $file -- It must be one of the following: ${VALID_PARAMS_FILE.join(',')}") + return result + } + + private static final Pattern PARAMS_VAR = ~/(?m)\$\{(\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)}/ + + protected String replaceVars0(String content, Map binding) { + content.replaceAll(PARAMS_VAR) { List matcher -> + // - the regex matcher is represented as list + // - the first element is the matching string ie. `${something}` + // - the second element is the group content ie. `something` + // - make sure the regex contains at least a group otherwise the closure + // parameter is a string instead of a list of the call fail + final placeholder = matcher.get(0) + final key = matcher.get(1) + + if( !binding.containsKey(key) ) + throw new AbortOperationException("Missing params file variable: $placeholder") + + return binding.get(key) + } + } + + private void readJsonFile(Path file, Map configVars, Map result) { + try { + final text = configVars ? replaceVars0(file.text, configVars) : file.text + final json = (Map) new JsonSlurper().parseText(text) + json.forEach((name, value) -> { + addParam0(result, name, value) + }) + } + catch( NoSuchFileException | FileNotFoundException e ) { + throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}") + } + catch( Exception e ) { + throw new AbortOperationException("Cannot parse params file: ${file.toUriString()} - Cause: ${e.message}", e) + } + } + + private void readYamlFile(Path file, Map configVars, Map result) { + try { + final text = configVars ? replaceVars0(file.text, configVars) : file.text + final yaml = (Map) new Yaml().load(text) + yaml.forEach((name, value) -> { + addParam0(result, name, value) + }) + } + catch( NoSuchFileException | FileNotFoundException e ) { + throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}") + } + catch( Exception e ) { + throw new AbortOperationException("Cannot parse params file: ${file.toUriString()}", e) + } + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index 4dfa78880e..b1f8165a20 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -21,6 +21,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowVariable import nextflow.Session import nextflow.exception.ScriptRuntimeException import nextflow.processor.PublishDir @@ -53,7 +54,7 @@ class PublishOp { private List indexRecords = [] - private volatile boolean complete + private DataflowVariable target PublishOp(Session session, String name, DataflowReadChannel source, Map opts) { this.session = session @@ -67,14 +68,13 @@ class PublishOp { this.indexOpts = new IndexOpts(session.outputDir, opts.index as Map) } - boolean getComplete() { complete } - - PublishOp apply() { + DataflowVariable apply() { final events = new HashMap(2) events.onNext = this.&onNext events.onComplete = this.&onComplete DataflowHelper.subscribeImpl(source, events) - return this + this.target = new DataflowVariable() + return target } /** @@ -223,7 +223,7 @@ class PublishOp { } log.trace "Publish operator complete" - this.complete = true + target.bind(indexPath ?: value) } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy index 376baa9253..136c84e86e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy @@ -48,8 +48,6 @@ abstract class BaseScript extends Script implements ExecutionContext { private OutputDef publisher - @Lazy InputStream stdin = { System.in }() - BaseScript() { meta = ScriptMeta.register(this) } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy index 2967e1c998..5700dd67de 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy @@ -16,11 +16,15 @@ package nextflow.script +import groovy.json.JsonBuilder +import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowVariable import nextflow.Session import nextflow.exception.ScriptRuntimeException import nextflow.extension.CH +import nextflow.extension.DumpHelper import nextflow.extension.PublishOp /** * Implements the DSL for publishing workflow outputs @@ -33,7 +37,7 @@ class OutputDsl { private Map declarations = [:] - private volatile List ops = [] + private Map output = [:] void declare(String name, Closure closure) { if( declarations.containsKey(name) ) @@ -70,7 +74,12 @@ class OutputDsl { final opts = publishOptions(name, defaults, overrides) if( opts.enabled == null || opts.enabled ) - ops << new PublishOp(session, name, CH.getReadChannel(source), opts).apply() + output[name] = new PublishOp(session, name, CH.getReadChannel(source), opts).apply() + } + + // write output to stdout as JSON + session.addIgniter { + println DumpHelper.prettyPrintJson(getOutput()) } } @@ -92,11 +101,8 @@ class OutputDsl { return opts } - boolean getComplete() { - for( final op : ops ) - if( !op.complete ) - return false - return true + Map getOutput() { + output.collectEntries { name, dv -> [name, dv.get()] } } static class DeclareDsl { diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy index d2c665938c..9cfbe94f24 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy @@ -18,6 +18,8 @@ package nextflow.script import java.nio.file.Path +import groovy.json.JsonSlurper +import groovy.yaml.YamlSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -25,6 +27,11 @@ import nextflow.Session import nextflow.file.FileHelper import nextflow.exception.ScriptRuntimeException import nextflow.script.types.Types +import nextflow.splitter.CsvSplitter +import nextflow.util.Duration +import nextflow.util.MemoryUnit +import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation +import org.codehaus.groovy.runtime.typehandling.GroovyCastException /** * Implements the DSL for defining workflow params * @@ -97,6 +104,18 @@ class ParamsDsl { if( str.isDouble() ) return str.toDouble() } + if( decl.type == Duration ) { + return Duration.of(str) + } + + if( decl.type == MemoryUnit ) { + return MemoryUnit.of(str) + } + + if( Collection.isAssignableFrom(decl.type) ) { + return resolveFromFile(decl.name, decl.type, FileHelper.asPath(str)) + } + if( decl.type == Path ) { return FileHelper.asPath(str) } @@ -108,12 +127,39 @@ class ParamsDsl { if( value == null ) return null - if( decl.type == Path && value instanceof CharSequence ) - return FileHelper.asPath(value.toString()) + if( value !instanceof CharSequence ) + return value + + final str = value.toString() + + if( Collection.isAssignableFrom(decl.type) ) + return resolveFromFile(decl.name, decl.type, FileHelper.asPath(str)) + + if( decl.type == Path ) + return FileHelper.asPath(str) return value } + private Object resolveFromFile(String name, Class type, Path file) { + final ext = file.getExtension() + final value = switch( ext ) { + case 'csv' -> new CsvSplitter().options(header: true, sep: ',').target(file).list() + case 'json' -> new JsonSlurper().parse(file) + case 'yaml' -> new YamlSlurper().parse(file) + case 'yml' -> new YamlSlurper().parse(file) + default -> throw new ScriptRuntimeException("Unrecognized file format '${ext}' for input file '${file}' supplied for parameter `${name}` -- should be CSV, JSON, or YAML") + } + + try { + return DefaultTypeTransformation.castToType(value, type) + } + catch( GroovyCastException e ) { + final actualType = value.getClass() + throw new ScriptRuntimeException("Parameter `${name}` with type ${Types.getName(type)} cannot be assigned to contents of '${file}' [${Types.getName(actualType)}]") + } + } + @Canonical private static class Param { String name diff --git a/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java b/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java index 2e5387c64a..c88b00fa64 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java +++ b/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java @@ -25,6 +25,7 @@ import org.codehaus.groovy.ast.Parameter; import org.codehaus.groovy.ast.expr.Expression; import org.codehaus.groovy.ast.expr.VariableExpression; +import org.codehaus.groovy.ast.stmt.BlockStatement; import org.codehaus.groovy.ast.stmt.EmptyStatement; import org.codehaus.groovy.ast.stmt.ExpressionStatement; import org.codehaus.groovy.ast.stmt.Statement; @@ -53,7 +54,7 @@ public WorkflowNode(String name, Parameter[] takes, Statement main, Statement em } public WorkflowNode(String name, Statement main) { - this(name, Parameter.EMPTY_ARRAY, main, EmptyStatement.INSTANCE, EmptyStatement.INSTANCE, EmptyStatement.INSTANCE, EmptyStatement.INSTANCE); + this(name, Parameter.EMPTY_ARRAY, main, EmptyStatement.INSTANCE, new BlockStatement(), EmptyStatement.INSTANCE, EmptyStatement.INSTANCE); } public boolean isEntry() { diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java index 4d3fd88a70..54f9e2ab4e 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java @@ -17,7 +17,9 @@ import java.lang.reflect.Modifier; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Set; import groovy.lang.groovydoc.GroovydocHolder; import nextflow.script.ast.ASTNodeMarker; @@ -26,6 +28,7 @@ import nextflow.script.ast.FunctionNode; import nextflow.script.ast.ImplicitClosureParameter; import nextflow.script.ast.IncludeNode; +import nextflow.script.ast.OutputBlockNode; import nextflow.script.ast.OutputNode; import nextflow.script.ast.ParamBlockNode; import nextflow.script.ast.ProcessNode; @@ -47,6 +50,7 @@ import org.codehaus.groovy.ast.ASTNode; import org.codehaus.groovy.ast.ClassHelper; import org.codehaus.groovy.ast.ClassNode; +import org.codehaus.groovy.ast.CodeVisitorSupport; import org.codehaus.groovy.ast.DynamicVariable; import org.codehaus.groovy.ast.FieldNode; import org.codehaus.groovy.ast.MethodNode; @@ -74,6 +78,7 @@ import org.codehaus.groovy.syntax.Types; import static nextflow.script.ast.ASTUtils.*; +import static org.codehaus.groovy.ast.tools.GeneralUtils.*; /** * Initialize the variable scopes for an AST. @@ -172,9 +177,79 @@ private void declareMethod(MethodNode mn) { public void visit() { var moduleNode = sourceUnit.getAST(); - if( moduleNode instanceof ScriptNode sn ) { - super.visit(sn); - vsc.checkUnusedVariables(); + if( !(moduleNode instanceof ScriptNode) ) + return; + var scriptNode = (ScriptNode) moduleNode; + + var entry = scriptNode.getEntry(); + if( entry != null && entry.isCodeSnippet() ) { + visitCodeSnippet(scriptNode); + return; + } + + super.visit(scriptNode); + vsc.checkUnusedVariables(); + } + + private void visitCodeSnippet(ScriptNode node) { + var entry = node.getEntry(); + var code = entry.main; + + // infer parameters from references + var paramDecls = new ParamsReferencesCollector().apply(code).stream() + .map(name -> new Parameter(ClassHelper.STRING_TYPE, name)) + .toArray(Parameter[]::new); + node.setParams(new ParamBlockNode(paramDecls)); + + // visit code + visitWorkflow(entry); + + // infer outputs from implicit variable declarations + var outputVars = asBlockStatements(code).stream() + .filter(stmt -> ( + stmt instanceof ExpressionStatement es + && es.getExpression() instanceof AssignmentExpression ae + && ae.getLeftExpression() instanceof VariableExpression + )) + .map((stmt) -> { + var es = (ExpressionStatement)stmt; + var ae = (AssignmentExpression)es.getExpression(); + var target = (VariableExpression)ae.getLeftExpression(); + return target; + }) + .toList(); + + var publishers = (BlockStatement) entry.publishers; + for( var outputVar : outputVars ) { + var name = outputVar.getName(); + var source = varX(name); + source.setAccessedVariable(outputVar); + var stmt = stmt(new AssignmentExpression(varX(name), source)); + publishers.addStatement(stmt); + } + + var outputDecls = outputVars.stream() + .map(outputVar -> new OutputNode(outputVar.getName(), ClassHelper.dynamicType(), new BlockStatement())) + .toList(); + node.setOutputs(new OutputBlockNode(outputDecls)); + } + + private static class ParamsReferencesCollector extends CodeVisitorSupport { + + private Set params; + + public Set apply(Statement node) { + params = new HashSet<>(); + visit(node); + return params; + } + + @Override + public void visitPropertyExpression(PropertyExpression node) { + super.visitPropertyExpression(node); + + if( node.getObjectExpression() instanceof VariableExpression ve && "params".equals(ve.getName()) ) + params.add(node.getPropertyAsString()); } }