Proper UTF-8 encoding and line endings on Windows (#5457)

Author: michelou

compiler/src/dotty/tools/dotc/decompiler/DecompilationPrinter.scala

@@ -3,6 +3,8 @@ package decompiler
 
 import java.io.{OutputStream, PrintStream}
 
+import scala.io.Codec
+
 import dotty.tools.dotc.core.Contexts._
 import dotty.tools.dotc.core.Phases.Phase
 import dotty.tools.dotc.core.tasty.TastyPrinter
@@ -24,8 +26,8 @@ class DecompilationPrinter extends Phase {
       var os: OutputStream = null
       var ps: PrintStream = null
       try {
-        os = File(outputDir.fileNamed("decompiled.scala").path).outputStream(append = true)
-        ps = new PrintStream(os)
+        os = File(outputDir.fileNamed("decompiled.scala").path)(Codec.UTF8).outputStream(append = true)
+        ps = new PrintStream(os, /* autoFlush = */ false, "UTF-8")
         printToOutput(ps)
       } finally {
         if (os ne null) os.close()

compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala

@@ -2,6 +2,8 @@ package dotty.tools
 package dotc
 package reporting
 
+import java.lang.System.{lineSeparator => EOL}
+
 import core.Contexts.Context
 import core.Decorators._
 import printing.Highlighting.{Blue, Red}
@@ -15,6 +17,7 @@ import scala.annotation.switch
 import scala.collection.mutable
 
 trait MessageRendering {
+
   /** Remove ANSI coloring from `str`, useful for getting real length of
     * strings
     *
@@ -101,7 +104,7 @@ trait MessageRendering {
 
     msg.linesIterator
       .map { line => " " * (offset - 1) + "|" + padding + line}
-      .mkString(sys.props("line.separator"))
+      .mkString(EOL)
   }
 
   /** The separator between errors containing the source file and error type
@@ -132,21 +135,21 @@ trait MessageRendering {
            |${Blue("Explanation")}
            |${Blue("===========")}"""
     )
-    sb.append('\n').append(m.explanation)
-    if (m.explanation.lastOption != Some('\n')) sb.append('\n')
+    sb.append(EOL).append(m.explanation)
+    if (m.explanation.lastOption != Some(EOL)) sb.append(EOL)
     sb.toString
   }
 
   /** The whole message rendered from `msg` */
   def messageAndPos(msg: Message, pos: SourcePosition, diagnosticLevel: String)(implicit ctx: Context): String = {
     val sb = mutable.StringBuilder.newBuilder
     val posString = posStr(pos, diagnosticLevel, msg)
-    if (posString.nonEmpty) sb.append(posString).append('\n')
+    if (posString.nonEmpty) sb.append(posString).append(EOL)
     if (pos.exists) {
       val (srcBefore, srcAfter, offset) = sourceLines(pos)
       val marker = columnMarker(pos, offset)
       val err = errorMsg(pos, msg.msg, offset)
-      sb.append((srcBefore ::: marker :: err :: outer(pos, " " * (offset - 1)) ::: srcAfter).mkString("\n"))
+      sb.append((srcBefore ::: marker :: err :: outer(pos, " " * (offset - 1)) ::: srcAfter).mkString(EOL))
     } else sb.append(msg.msg)
     sb.toString
   }

compiler/src/dotty/tools/repl/ReplDriver.scala

@@ -1,6 +1,6 @@
 package dotty.tools.repl
 
-import java.io.PrintStream
+import java.io.{File => JFile, PrintStream}
 
 import dotty.tools.dotc.ast.Trees._
 import dotty.tools.dotc.ast.{tpd, untpd}
@@ -324,9 +324,9 @@ class ReplDriver(settings: Array[String],
       state
 
     case Load(path) =>
-      val file = new java.io.File(path)
+      val file = new JFile(path)
       if (file.exists) {
-        val contents = scala.io.Source.fromFile(file).mkString
+        val contents = scala.io.Source.fromFile(file, "UTF-8").mkString
         run(contents)
       }
       else {

compiler/test/dotty/tools/repl/ReplCompilerTests.scala

@@ -1,5 +1,7 @@
 package dotty.tools.repl
 
+import java.lang.System.{lineSeparator => EOL}
+
 import org.junit.Assert._
 import org.junit.{Ignore, Test}
 
@@ -51,7 +53,7 @@ class ReplCompilerTests extends ReplTest {
       "val res1: Int = 20"
     )
 
-    assertEquals(expected, storedOutput().split("\n").toList)
+    assertEquals(expected, storedOutput().split(EOL).toList)
   }
 
   @Test def testImportMutable =
@@ -122,6 +124,6 @@ class ReplCompilerTests extends ReplTest {
     )
 
     run(source)
-    assertEquals(expected, storedOutput().split("\n").toList)
+    assertEquals(expected, storedOutput().split(EOL).toList)
   }
 }

compiler/test/dotty/tools/repl/ScriptedTests.scala

@@ -1,7 +1,8 @@
 package dotty.tools
 package repl
 
-import java.io.{ File => JFile }
+import java.io.{File => JFile}
+import java.lang.System.{lineSeparator => EOL}
 
 import org.junit.Assert._
 import org.junit.Test
@@ -22,7 +23,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
 
   private def testFile(f: JFile): Unit = {
     val prompt = "scala>"
-    val lines = Source.fromFile(f).getLines().buffered
+    val lines = Source.fromFile(f, "UTF-8").getLines().buffered
 
     assert(lines.head.startsWith(prompt),
       s"""Each file has to start with the prompt: "$prompt"""")
@@ -44,7 +45,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
     def evaluate(state: State, input: String, prompt: String) =
       try {
         val nstate = run(input.drop(prompt.length))(state)
-        val out = input + "\n" + storedOutput()
+        val out = input + EOL + storedOutput()
         (out, nstate)
       }
       catch {
@@ -60,7 +61,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
       }
 
     val expectedOutput =
-      Source.fromFile(f).getLines().flatMap(filterEmpties).mkString("\n")
+      Source.fromFile(f, "UTF-8").getLines().flatMap(filterEmpties).mkString(EOL)
     val actualOutput = {
       resetToInitial()
       val inputRes = extractInputs(prompt)
@@ -70,7 +71,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
         buf.append(out)
         nstate
       }
-      buf.flatMap(filterEmpties).mkString("\n")
+      buf.flatMap(filterEmpties).mkString(EOL)
     }
 
     if (expectedOutput != actualOutput) {

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -3,26 +3,26 @@ package tools
 package vulpix
 
 import java.io.{File => JFile}
-import java.text.SimpleDateFormat
-import java.util.HashMap
+import java.lang.System.{lineSeparator => EOL}
 import java.nio.file.StandardCopyOption.REPLACE_EXISTING
 import java.nio.file.{Files, NoSuchFileException, Path, Paths}
+import java.text.SimpleDateFormat
+import java.util.{HashMap, Timer, TimerTask}
 import java.util.concurrent.{TimeUnit, TimeoutException, Executors => JExecutors}
-import java.util.{Timer, TimerTask}
 
+import scala.collection.mutable
 import scala.io.Source
+import scala.util.{Random, Try}
 import scala.util.control.NonFatal
-import scala.util.Try
-import scala.collection.mutable
 import scala.util.matching.Regex
-import scala.util.Random
+
+import dotc.{Compiler, Driver}
 import dotc.core.Contexts._
+import dotc.decompiler
+import dotc.interfaces.Diagnostic.ERROR
 import dotc.reporting.{Reporter, TestReporter}
 import dotc.reporting.diagnostic.MessageContainer
-import dotc.interfaces.Diagnostic.ERROR
 import dotc.util.DiffUtil
-import dotc.{Compiler, Driver}
-import dotc.decompiler
 import dotty.tools.vulpix.TestConfiguration.defaultOptions
 
 /** A parallel testing suite whose goal is to integrate nicely with JUnit
@@ -535,16 +535,16 @@ trait ParallelTesting extends RunnerOrchestration { self =>
                     val ignoredFilePathLine = "/** Decompiled from"
                     val stripTrailingWhitespaces = "(.*\\S|)\\s+".r
                     val output = Source.fromFile(outDir.getParent + "_decompiled" + JFile.separator + outDir.getName
-                      + JFile.separator + "decompiled.scala").getLines().map {line =>
+                      + JFile.separator + "decompiled.scala", "UTF-8").getLines().map {line =>
                       stripTrailingWhitespaces.unapplySeq(line).map(_.head).getOrElse(line)
                     }.toList
 
-                    val check: String = Source.fromFile(checkFile).getLines().filter(!_.startsWith(ignoredFilePathLine))
-                      .mkString("\n")
+                    val check: String = Source.fromFile(checkFile, "UTF-8").getLines().filter(!_.startsWith(ignoredFilePathLine))
+                      .mkString(EOL)
 
-                    if (output.filter(!_.startsWith(ignoredFilePathLine)).mkString("\n") != check) {
+                    if (output.filter(!_.startsWith(ignoredFilePathLine)).mkString(EOL) != check) {
                       val outFile = dotty.tools.io.File(checkFile.toPath).addExtension(".out")
-                      outFile.writeAll(output.mkString("\n"))
+                      outFile.writeAll(output.mkString(EOL))
                       val msg =
                         s"""Output differed for test $name, use the following command to see the diff:
                            |  > diff $checkFile $outFile
@@ -617,7 +617,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success!
         case Success(output) => {
           val outputLines = output.linesIterator.toArray :+ DiffUtil.EOF
-          val checkLines: Array[String] = Source.fromFile(checkFile.get).getLines().toArray :+ DiffUtil.EOF
+          val checkLines: Array[String] = Source.fromFile(checkFile.get, "UTF-8").getLines().toArray :+ DiffUtil.EOF
           val sourceTitle = testSource.title
 
           def linesMatch =
@@ -726,7 +726,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
           val errorMap = new HashMap[String, Integer]()
           var expectedErrors = 0
           files.filter(_.getName.endsWith(".scala")).foreach { file =>
-            Source.fromFile(file).getLines().zipWithIndex.foreach { case (line, lineNbr) =>
+            Source.fromFile(file, "UTF-8").getLines().zipWithIndex.foreach { case (line, lineNbr) =>
               val errors = line.sliding("// error".length).count(_.mkString == "// error")
               if (errors > 0)
                 errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors)