[Testament] Extend and document message testing aids

doc/testament.md

@@ -113,7 +113,7 @@ Example "template" **to edit** and write a Testament unittest:
     # Provide an `output` string to assert that the test prints to standard out
     # exactly the expected string. Provide an `outputsub` string to assert that
     # the string given here is a substring of the standard out output of the
-    # test.
+    # test (the output includes both the compiler and test execution output).
     output: ""
     outputsub: ""
 
@@ -153,8 +153,7 @@ Example "template" **to edit** and write a Testament unittest:
     # Command the test should use to run. If left out or an empty string is
     # provided, the command is taken to be:
     # "nim $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file"
-    # You can use the $target, $options, and $file placeholders in your own
-    # command, too.
+    # Subject to variable interpolation.
     cmd: "nim c -r $file"
 
     # Maximum generated temporary intermediate code file size for the test.
@@ -189,7 +188,76 @@ Example "template" **to edit** and write a Testament unittest:
 * `This is not the full spec of Testament, check the Testament Spec on GitHub, see parseSpec(). <https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L238>`_
 * `Nim itself uses Testament, so there are plenty of test examples. <https://github.com/nim-lang/Nim/tree/devel/tests>`_
 * Has some built-in CI compatibility, like Azure Pipelines, etc.
-* `Testament supports inlined error messages on Unittests, basically comments with the expected error directly on the code. <https://github.com/nim-lang/Nim/blob/9a110047cbe2826b1d4afe63e3a1f5a08422b73f/tests/effects/teffects1.nim>`_
+
+
+Inline hints, warnings and errors (notes)
+-----------------------------------------
+
+Testing the line, column, kind and message of hints, warnings and errors can
+be written inline like so:
+
+.. code-block:: nim
+
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+
+The opening `#[tt.` marks the message line.
+The `^` marks the message column.
+
+Inline messages can be combined with `nimout` when `nimoutFull` is false (default).
+This allows testing for expected messages from other modules:
+
+.. code-block:: nim
+
+  discard """
+    nimout: "config.nims(1, 1) Hint: some hint message [User]"
+  """
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+
+Multiple messages for a line can be checked by delimiting messages with ';':
+
+.. code-block:: nim
+
+  discard """
+    matrix: "--errorMax:0 --styleCheck:error"
+  """
+
+  proc generic_proc*[T](a_a: int) = #[tt.Error
+       ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                        ^ 'a_a' should be: 'aA' ]#
+    discard
+
+Use `--errorMax:0` in `matrix`, or `cmd: "nim check $file"` when testing
+for multiple 'Error' messages.
+
+Output message variable interpolation
+-------------------------------------
+
+`errormsg`, `nimout`, and inline messages are subject to these variable interpolations:
+
+* `${/}` - platform's directory separator
+* `$file` - the filename (without directory) of the test
+
+All other `$` characters need escaped as `$$`.
+
+Cmd variable interpolation
+--------------------------
+
+The `cmd` option is subject to these variable interpolations:
+
+* `$target` - the compilation target, e.g. `c`.
+* `$options` - the options for the compiler.
+* `$file` - the file path of the test.
+* `$filedir` - the directory of the test file.
+
+.. code-block:: nim
+
+  discard """
+    cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
+  """
+
+All other `$` characters need escaped as `$$`.
 
 
 Unit test Examples

testament/specs.nim

@@ -75,6 +75,7 @@ type
     # xxx make sure `isJoinableSpec` takes into account each field here.
     action*: TTestAction
     file*, cmd*: string
+    filename*: string ## Test filename (without path).
     input*: string
     outputCheck*: TOutputCheck
     sortoutput*: bool
@@ -127,19 +128,56 @@ when not declared(parseCfgBool):
     of "n", "no", "false", "0", "off": result = false
     else: raise newException(ValueError, "cannot interpret as a bool: " & s)
 
+proc addLine*(self: var string; pieces: varargs[string]) =
+  for piece in pieces:
+    self.add piece
+  self.add "\n"
+
+
 const
-  inlineErrorMarker = "#[tt."
+  inlineErrorKindMarker = "tt."
+  inlineErrorMarker = "#[" & inlineErrorKindMarker
 
 proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
+  ## Extract inline error messages.
+  ##
+  ## Can parse a single message for a line:
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name] ]#
+  ##
+  ## Can parse multiple messages for a line when they are separated by ';':
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error
+  ##                           ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error
+  ##                                       ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name];
+  ##     tt.Error              ^ 'no_destroy' should be: 'nodestroy' [Name];
+  ##     tt.Error                          ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##
   result = i + len(inlineErrorMarker)
   inc col, len(inlineErrorMarker)
+  let msgLine = line
+  var msgCol = -1
+  var msg = ""
   var kind = ""
-  while result < s.len and s[result] in IdentChars:
-    kind.add s[result]
-    inc result
-    inc col
 
-  var caret = (line, -1)
+  template parseKind =
+    while result < s.len and s[result] in IdentChars:
+      kind.add s[result]
+      inc result
+      inc col
+    if kind notin ["Hint", "Warning", "Error"]:
+      spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error"
 
   template skipWhitespace =
     while result < s.len and s[result] in Whitespace:
@@ -150,34 +188,70 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T
         inc col
       inc result
 
+  template parseCaret =
+    if result < s.len and s[result] == '^':
+      msgCol = col
+      inc result
+      inc col
+      skipWhitespace()
+    else:
+      spec.parseErrors.addLine "expected column marker ('^') for inline message"
+
+  template isMsgDelimiter: bool =
+    s[result] == ';' and
+    (block:
+      let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1)
+      if s.len > nextTokenIdx + len(inlineErrorKindMarker) and
+         s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker:
+        true
+      else:
+        false)
+
+  template trimTrailingMsgWhitespace =
+    while msg.len > 0 and msg[^1] in Whitespace:
+      setLen msg, msg.len - 1
+
+  template addInlineError =
+    doAssert msg[^1] notin Whitespace
+    if kind == "Error": spec.action = actionReject
+    spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol)
+
+  parseKind()
   skipWhitespace()
-  if result < s.len and s[result] == '^':
-    caret = (line-1, col)
-    inc result
-    inc col
-    skipWhitespace()
+  parseCaret()
 
-  var msg = ""
   while result < s.len-1:
     if s[result] == '\n':
+      msg.add '\n'
       inc result
       inc line
       col = 1
-    elif s[result] == ']' and s[result+1] == '#':
-      while msg.len > 0 and msg[^1] in Whitespace:
-        setLen msg, msg.len - 1
-
+    elif isMsgDelimiter():
+      trimTrailingMsgWhitespace()
       inc result
+      skipWhitespace()
+      addInlineError()
+      inc result, len(inlineErrorKindMarker)
+      inc col, 1 + len(inlineErrorKindMarker)
+      kind.setLen 0
+      msg.setLen 0
+      parseKind()
+      skipWhitespace()
+      parseCaret()
+    elif s[result] == ']' and s[result+1] == '#':
+      trimTrailingMsgWhitespace()
+      inc result, 2
       inc col, 2
-      if kind == "Error": spec.action = actionReject
-      spec.unjoinable = true
-      spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1])
+      addInlineError()
       break
     else:
       msg.add s[result]
       inc result
       inc col
 
+  if spec.inlineErrors.len > 0:
+    spec.unjoinable = true
+
 proc extractSpec(filename: string; spec: var TSpec): string =
   const
     tripleQuote = "\"\"\""
@@ -229,15 +303,6 @@ proc parseTargets*(value: string): set[TTarget] =
     of "js": result.incl(targetJS)
     else: raise newException(ValueError, "invalid target: '$#'" % v)
 
-proc addLine*(self: var string; a: string) =
-  self.add a
-  self.add "\n"
-
-proc addLine*(self: var string; a, b: string) =
-  self.add a
-  self.add b
-  self.add "\n"
-
 proc initSpec*(filename: string): TSpec =
   result.file = filename
 
@@ -249,6 +314,7 @@ proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool =
 
 proc parseSpec*(filename: string): TSpec =
   result.file = filename
+  result.filename = extractFilename(filename)
   let specStr = extractSpec(filename, result)
   var ss = newStringStream(specStr)
   var p: CfgParser
@@ -439,3 +505,15 @@ proc parseSpec*(filename: string): TSpec =
   result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
   if not result.inCurrentBatch:
     result.err = reDisabled
+
+  # Interpolate variables in msgs:
+  template varSub(msg: string): string =
+    try:
+      msg % ["/", $DirSep, "file", result.filename]
+    except ValueError:
+      result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unitests-output-message-variable-interpolation')"
+      msg
+  result.nimout = result.nimout.varSub
+  result.msg = result.msg.varSub
+  for inlineError in result.inlineErrors.mitems:
+    inlineError.msg = inlineError.msg.varSub