Skip to content

Commit c15d091

Browse files
HyukjinKwonRobert Kruszewski
authored andcommitted
[SPARK-18685][TESTS] Fix URI and release resources after opening in tests at ExecutorClassLoaderSuite
## What changes were proposed in this pull request? This PR fixes two problems as below: - Close `BufferedSource` after `Source.fromInputStream(...)` to release resource and make the tests pass on Windows in `ExecutorClassLoaderSuite` ``` [info] Exception encountered when attempting to run a suite with class name: org.apache.spark.repl.ExecutorClassLoaderSuite *** ABORTED *** (7 seconds, 333 milliseconds) [info] java.io.IOException: Failed to delete: C:\projects\spark\target\tmp\spark-77b2f37b-6405-47c4-af1c-4a6a206511f2 [info] at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:1010) [info] at org.apache.spark.repl.ExecutorClassLoaderSuite.afterAll(ExecutorClassLoaderSuite.scala:76) [info] at org.scalatest.BeforeAndAfterAll$class.afterAll(BeforeAndAfterAll.scala:213) ... ``` - Fix URI correctly so that related tests can be passed on Windows. ``` [info] - child first *** FAILED *** (78 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - parent first *** FAILED *** (15 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - child first can fall back *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - child first can fail *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - resource from parent *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - resources from parent *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ``` ## How was this patch tested? Manually tested via AppVeyor. **Before** https://ci.appveyor.com/project/spark-test/spark/build/102-rpel-ExecutorClassLoaderSuite **After** https://ci.appveyor.com/project/spark-test/spark/build/108-rpel-ExecutorClassLoaderSuite Author: hyukjinkwon <[email protected]> Closes apache#16116 from HyukjinKwon/close-after-open.
1 parent 92c344c commit c15d091

File tree

1 file changed

+17
-8
lines changed

1 file changed

+17
-8
lines changed

repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import java.nio.charset.StandardCharsets
2424
import java.nio.file.{Paths, StandardOpenOption}
2525
import java.util
2626

27-
import scala.concurrent.duration._
2827
import scala.io.Source
2928
import scala.language.implicitConversions
3029

@@ -34,8 +33,6 @@ import org.mockito.Mockito._
3433
import org.mockito.invocation.InvocationOnMock
3534
import org.mockito.stubbing.Answer
3635
import org.scalatest.BeforeAndAfterAll
37-
import org.scalatest.concurrent.Interruptor
38-
import org.scalatest.concurrent.Timeouts._
3936
import org.scalatest.mock.MockitoSugar
4037

4138
import org.apache.spark._
@@ -61,7 +58,7 @@ class ExecutorClassLoaderSuite
6158
super.beforeAll()
6259
tempDir1 = Utils.createTempDir()
6360
tempDir2 = Utils.createTempDir()
64-
url1 = "file://" + tempDir1
61+
url1 = tempDir1.toURI.toURL.toString
6562
urls2 = List(tempDir2.toURI.toURL).toArray
6663
childClassNames.foreach(TestUtils.createCompiledClass(_, tempDir1, "1"))
6764
parentResourceNames.foreach { x =>
@@ -118,8 +115,14 @@ class ExecutorClassLoaderSuite
118115
val resourceName: String = parentResourceNames.head
119116
val is = classLoader.getResourceAsStream(resourceName)
120117
assert(is != null, s"Resource $resourceName not found")
121-
val content = Source.fromInputStream(is, "UTF-8").getLines().next()
122-
assert(content.contains("resource"), "File doesn't contain 'resource'")
118+
119+
val bufferedSource = Source.fromInputStream(is, "UTF-8")
120+
Utils.tryWithSafeFinally {
121+
val content = bufferedSource.getLines().next()
122+
assert(content.contains("resource"), "File doesn't contain 'resource'")
123+
} {
124+
bufferedSource.close()
125+
}
123126
}
124127

125128
test("resources from parent") {
@@ -128,8 +131,14 @@ class ExecutorClassLoaderSuite
128131
val resourceName: String = parentResourceNames.head
129132
val resources: util.Enumeration[URL] = classLoader.getResources(resourceName)
130133
assert(resources.hasMoreElements, s"Resource $resourceName not found")
131-
val fileReader = Source.fromInputStream(resources.nextElement().openStream()).bufferedReader()
132-
assert(fileReader.readLine().contains("resource"), "File doesn't contain 'resource'")
134+
135+
val bufferedSource = Source.fromInputStream(resources.nextElement().openStream())
136+
Utils.tryWithSafeFinally {
137+
val fileReader = bufferedSource.bufferedReader()
138+
assert(fileReader.readLine().contains("resource"), "File doesn't contain 'resource'")
139+
} {
140+
bufferedSource.close()
141+
}
133142
}
134143

135144
test("fetch classes using Spark's RpcEnv") {

0 commit comments

Comments
 (0)