Script parser (#4613)

---------

Signed-off-by: Ben Sherman <[email protected]>
Signed-off-by: Paolo Di Tommaso <[email protected]>
Co-authored-by: Paolo Di Tommaso <[email protected]>

Author: bentsherman

docs/reference/env-vars.md

@@ -185,7 +185,7 @@ The following environment variables control the configuration of the Nextflow ru
 `NXF_SYNTAX_PARSER`
 : :::{versionadded} 25.02.0-edge
   :::
-: Set to `'v2'` to use the {ref}`strict syntax <updating-syntax-page>` for Nextflow config files (default: `'v1'`).
+: Set to `'v2'` to use the {ref}`strict syntax <updating-syntax-page>` for Nextflow scripts and config files (default: `'v1'`).
 
 `NXF_TEMP`
 : Directory where temporary files are stored

docs/updating-syntax.md

@@ -8,13 +8,17 @@ This page explains how to update Nextflow scripts and config files to adhere to
 If you are still using DSL1, see {ref}`dsl1-page` to learn how to migrate your Nextflow pipelines to DSL2 before consulting this guide.
 :::
 
+(strict-syntax)=
+
 ## Preparing for strict syntax
 
-The strict syntax is a subset of DSL2. While DSL2 allows any Groovy syntax, the strict syntax allows only a subset of Groovy syntax for Nextflow scripts and config files. This new specification enables more specific error reporting, ensures more consistent code, and will allow the Nextflow language to evolve independently of Groovy.
+:::{versionadded} 25.02.0-edge
+The strict syntax can be enabled in Nextflow by setting `NXF_SYNTAX_PARSER=v2`.
+:::
 
-The strict syntax is currently only enforced by the Nextflow language server, which is provided as part of the Nextflow {ref}`vscode-page`. However, the strict syntax will be gradually adopted by the Nextflow CLI in future releases and will eventually be the only way to write Nextflow code.
+The strict syntax is a subset of DSL2. While DSL2 allows any Groovy syntax, the strict syntax allows only a subset of Groovy syntax for Nextflow scripts and config files. This new specification enables more specific error reporting, ensures more consistent code, and will allow the Nextflow language to evolve independently of Groovy.
 
-New language features will be implemented as part of the strict syntax, and not the current DSL2 parser, with few exceptions. Therefore, it is important to prepare for the strict syntax in order to use new language features in the future.
+The strict syntax will eventually become the only way to write Nextflow code, and new language features will be implemented only in the strict syntax, with few exceptions. Therefore, it is important to prepare for the strict syntax in order to maintain compatibility with future versions of Nextflow and be able to use new language features.
 
 This section describes the key differences between the DSL2 and the strict syntax. In general, the amount of changes that are required depends on the amount of custom Groovy code in your scripts and config files.
 

docs/vscode.md

@@ -102,6 +102,8 @@ The following settings are available:
 `nextflow.paranoidWarnings`
 : Enable additional warnings for future deprecations, potential problems, and other discouraged patterns.
 
+(vscode-language-server)=
+
 ## Language server
 
 Most of the functionality of the VS Code extension is provided by the [Nextflow language server](https://github.com/nextflow-io/language-server), which implements the [Language Server Protocol (LSP)](https://microsoft.github.io/language-server-protocol/) for Nextflow scripts and config files.

modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy

@@ -356,44 +356,27 @@ class NextflowDSLImpl implements ASTTransformation {
             MethodCallExpression callx
             VariableExpression varx
 
-            if( (callx=isMethodCallX(stat.expression)) && isThisX(callx.objectExpression) ) {
-                final name = "_${type}_${callx.methodAsString}"
-                return stmt( callThisX(name, callx.arguments) )
-            }
-
             if( (varx=isVariableX(stat.expression)) ) {
-                final name = "_${type}_${varx.name}"
-                return stmt( callThisX(name) )
+                return stmt( callThisX("_${type}_", args(constX(varx.name))) )
             }
 
             if( type == WORKFLOW_EMIT ) {
                 return createAssignX(stat, body, type, uniqueNames)
             }
 
-            syntaxError(stat, "Workflow malformed parameter definition")
+            syntaxError(stat, "Invalid workflow ${type}")
             return stat
         }
 
         protected Statement createAssignX(ExpressionStatement stat, List<Statement> body, String type, Set<String> uniqueNames) {
             BinaryExpression binx
-            MethodCallExpression callx
-            Expression args=null
 
             if( (binx=isAssignX(stat.expression)) ) {
                 // keep the statement in body to allow it to be evaluated
                 body.add(stat)
                 // and create method call expr to capture the var name in the emission
                 final left = (VariableExpression)binx.leftExpression
-                final name = "_${type}_${left.name}"
-                return stmt( callThisX(name) )
-            }
-
-            if( (callx=isMethodCallX(stat.expression)) && callx.objectExpression.text!='this' && hasTo(callx)) {
-                // keep the args
-                args = callx.arguments
-                // replace the method call expression with a property
-                stat.expression = new PropertyExpression(callx.objectExpression, callx.method)
-                // then, fallback to default case
+                return stmt( callThisX("_${type}_", args(constX(left.name))) )
             }
 
             // wrap the expression into a assignment expression
@@ -405,19 +388,7 @@ class NextflowDSLImpl implements ASTTransformation {
             body.add(stmt(assign))
 
             // the call method statement for the emit declaration
-            final name="_${type}_${var}"
-            callx =  args ? callThisX(name, args) : callThisX(name)
-            return stmt(callx)
-        }
-
-        protected boolean hasTo(MethodCallExpression callX) {
-            def tupleX = isTupleX(callX.arguments)
-            if( !tupleX ) return false
-            if( !tupleX.expressions ) return false
-            def mapX = isMapX(tupleX.expressions[0])
-            if( !mapX ) return false
-            def entry = mapX.getMapEntryExpressions().find { isConstX(it.keyExpression).text=='to' }
-            return entry != null
+            return stmt( callThisX("_${type}_", args(constX(var))) )
         }
 
         protected String getNextName(Set<String> allNames) {

modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy

@@ -16,10 +16,6 @@
 
 package nextflow.script
 
-import nextflow.exception.ScriptCompilationException
-import nextflow.plugin.extension.PluginExtensionProvider
-import nextflow.plugin.Plugins
-
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 
@@ -32,6 +28,10 @@ import groovy.util.logging.Slf4j
 import nextflow.NF
 import nextflow.Session
 import nextflow.exception.IllegalModulePath
+import nextflow.exception.ScriptCompilationException
+import nextflow.plugin.Plugins
+import nextflow.plugin.extension.PluginExtensionProvider
+import nextflow.script.parser.v1.ScriptLoaderV1
 /**
  * Implements a script inclusion
  *
@@ -96,11 +96,11 @@ class IncludeDef {
         return this
     }
 
-    /*
-     * Note: this method invocation is injected during the Nextflow AST manipulation.
-     * Do not use it explicitly.
+    /**
+     * Used internally by the script DSL to include modules
+     * into a script.
      *
-     * @param ownerParams The params in the owner context
+     * @param ownerParams The params in the including script context
      */
     void load0(ScriptBinding.ParamsMap ownerParams) {
         checkValidPath(path)
@@ -109,7 +109,7 @@ class IncludeDef {
             return
         }
         // -- resolve the concrete against the current script
-        final moduleFile = realModulePath(path)
+        final moduleFile = realModulePath(path).normalize()
         // -- load the module
         final moduleScript = loadModule0(moduleFile, resolveParams(ownerParams), session)
         // -- add it to the inclusions
@@ -136,10 +136,17 @@ class IncludeDef {
     @PackageScope
     @Memoized
     static BaseScript loadModule0(Path path, Map params, Session session) {
+        final script = ScriptMeta.getScriptByPath(path)
+        if( script ) {
+            script.getBinding().setParams(params)
+            script.run()
+            return script
+        }
+
         final binding = new ScriptBinding() .setParams(params)
 
         // the execution of a library file has as side effect the registration of declared processes
-        new ScriptParser(session)
+        new ScriptLoaderV1(session)
                 .setModule(true)
                 .setBinding(binding)
                 .runScript(path)

modules/nextflow/src/main/groovy/nextflow/script/ScriptLoader.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script
+
+import java.nio.file.Path
+
+/**
+ * Interface for parsing and executing a Nextflow script.
+ *
+ * @author Ben Sherman <[email protected]>
+ */
+interface ScriptLoader {
+
+    ScriptLoader setEntryName(String name);
+
+    ScriptLoader setModule(boolean value);
+
+    BaseScript getScript();
+
+    Object getResult();
+
+    ScriptLoader parse(Path scriptPath);
+
+    ScriptLoader runScript();
+
+}

modules/nextflow/src/main/groovy/nextflow/script/ScriptLoaderFactory.groovy

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.script
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.Session
+import nextflow.SysEnv
+import nextflow.script.parser.v1.ScriptLoaderV1
+import nextflow.script.parser.v2.ScriptLoaderV2
+
+/**
+ * Factory for creating an instance of {@link ScriptLoader}.
+ *
+ * @author Ben Sherman <[email protected]>
+ */
+@Slf4j
+@CompileStatic
+class ScriptLoaderFactory {
+
+    static ScriptLoader create(Session session) {
+        final parser = SysEnv.get('NXF_SYNTAX_PARSER', 'v1')
+        if( parser == 'v1' ) {
+            return new ScriptLoaderV1(session)
+        }
+        if( parser == 'v2' ) {
+            log.debug "Using script parser v2"
+            return new ScriptLoaderV2(session)
+        }
+        throw new IllegalStateException("Invalid NXF_SYNTAX_PARSER setting -- should be either 'v1' or 'v2'")
+    }
+
+}

modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy

@@ -48,11 +48,14 @@ class ScriptMeta {
 
     static private Map<BaseScript,ScriptMeta> REGISTRY = new HashMap<>(10)
 
+    static private Map<Path,BaseScript> scriptsByPath = new HashMap<>(10)
+
     static private Set<String> resolvedProcessNames = new HashSet<>(20)
 
     @TestOnly
     static void reset() {
         REGISTRY.clear()
+        scriptsByPath.clear()
         resolvedProcessNames.clear()
     }
 
@@ -61,6 +64,10 @@ class ScriptMeta {
         return REGISTRY.get(script)
     }
 
+    static BaseScript getScriptByPath(Path path) {
+        return scriptsByPath.get(path)
+    }
+
     static Set<String> allProcessNames() {
         def result = new HashSet()
         for( ScriptMeta entry : REGISTRY.values() )
@@ -94,8 +101,8 @@ class ScriptMeta {
         get(ExecutionStack.script())
     }
 
-    /** the script {@link Class} object */
-    private Class<? extends BaseScript> clazz
+    /** the script object */
+    private BaseScript script
 
     /** The location path from where the script has been loaded */
     private Path scriptPath
@@ -115,12 +122,12 @@ class ScriptMeta {
 
     Path getModuleDir () { scriptPath?.parent }
 
-    String getScriptName() { clazz.getName() }
+    String getScriptName() { script.getClass().getName() }
 
     boolean isModule() { module }
 
     ScriptMeta(BaseScript script) {
-        this.clazz = script.class
+        this.script = script
         for( def entry : definedFunctions0(script) ) {
             addDefinition(entry)
         }
@@ -129,12 +136,11 @@ class ScriptMeta {
     /** only for testing */
     protected ScriptMeta() {}
 
-    @PackageScope
     void setScriptPath(Path path) {
+        scriptsByPath.put(path, script)
         scriptPath = path
     }
 
-    @PackageScope
     void setModule(boolean val) {
         this.module = val
     }

modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy

@@ -47,7 +47,7 @@ class ScriptRunner {
     /**
      * The script interpreter
      */
-    private ScriptParser scriptParser
+    private ScriptLoader scriptLoader
 
     /**
      * The pipeline file (it may be null when it's provided as string)
@@ -104,7 +104,7 @@ class ScriptRunner {
     /**
      * @return The interpreted script object
      */
-    @Deprecated BaseScript getScriptObj() { scriptParser.script }
+    @Deprecated BaseScript getScriptObj() { scriptLoader.getScript() }
 
     /**
      * @return The result produced by the script execution
@@ -225,12 +225,12 @@ class ScriptRunner {
     }
 
     protected void parseScript( ScriptFile scriptFile, String entryName ) {
-        scriptParser = new ScriptParser(session)
+        scriptLoader = ScriptLoaderFactory.create(session)
                             .setEntryName(entryName)
                             // setting module true when running in "inspect" mode to prevent the running the entry workflow
                             .setModule(ContainerInspectMode.active())
                             .parse(scriptFile.main)
-        session.script = scriptParser.script
+        session.script = scriptLoader.getScript()
     }
 
 
@@ -241,11 +241,11 @@ class ScriptRunner {
      */
     protected run() {
         log.debug "> Launching execution"
-        assert scriptParser, "Missing script instance to run"
+        assert scriptLoader, "Missing script instance to run"
         // -- launch the script execution
-        scriptParser.runScript()
+        scriptLoader.runScript()
         // -- normalise output
-        result = normalizeOutput(scriptParser.getResult())
+        result = normalizeOutput(scriptLoader.getResult())
         // -- ignite dataflow network
         session.fireDataflowNetwork(preview)
     }