@@ -79,63 +79,57 @@ class SparkContextSuite extends FunSuite with LocalSparkContext {
7979 val byteArray2 = converter.convert(bytesWritable)
8080 assert(byteArray2.length === 0 )
8181 }
82-
83- test(" addFile works" ) {
84- val file = File .createTempFile(" someprefix" , " somesuffix" )
85- val absolutePath = file.getAbsolutePath
86- try {
87- Files .write(" somewords" , file, UTF_8 )
88- val length = file.length()
89- sc = new SparkContext (new SparkConf ().setAppName(" test" ).setMaster(" local" ))
90- sc.addFile(file.getAbsolutePath)
91- sc.parallelize(Array (1 ), 1 ).map(x => {
92- val gotten = new File (SparkFiles .get(file.getName))
93- if (! gotten.exists()) {
94- throw new SparkException (" file doesn't exist" )
95- }
96- if (length != gotten.length()) {
97- throw new SparkException (
98- s " file has different length $length than added file ${gotten.length()}" )
99- }
100- if (absolutePath == gotten.getAbsolutePath) {
101- throw new SparkException (" file should have been copied" )
102- }
103- x
104- }).count()
105- } finally {
106- sc.stop()
107- }
108- }
10982
110- test(" addFile works with relative path" ) {
83+ test(" addFile works" ) {
84+ val file1 = File .createTempFile(" someprefix1" , " somesuffix1" )
85+ val absolutePath1 = file1.getAbsolutePath
86+
11187 val pluto = Utils .createTempDir()
112- val file = File .createTempFile(" someprefix" , " somesuffix" , pluto)
113- val relativePath = file.getParent + " /../" + file.getParentFile.getName + " /" + file.getName
114- val absolutePath = file.getAbsolutePath
88+ val file2 = File .createTempFile(" someprefix2" , " somesuffix2" , pluto)
89+ val relativePath = file2.getParent + " /../" + file2.getParentFile.getName + " /" + file2.getName
90+ val absolutePath2 = file2.getAbsolutePath
91+
11592 try {
116- Files .write(" somewords" , file, UTF_8 )
117- val length = file.length()
93+ Files .write(" somewords1" , file1, UTF_8 )
94+ Files .write(" somewords2" , file2, UTF_8 )
95+ val length1 = file1.length()
96+ val length2 = file2.length()
97+
11898 sc = new SparkContext (new SparkConf ().setAppName(" test" ).setMaster(" local" ))
99+ sc.addFile(file1.getAbsolutePath)
119100 sc.addFile(relativePath)
120101 sc.parallelize(Array (1 ), 1 ).map(x => {
121- val gotten = new File (SparkFiles .get(file.getName))
122- if (! gotten.exists()) {
123- throw new SparkException (" file doesn't exist" )
102+ val gotten1 = new File (SparkFiles .get(file1.getName))
103+ val gotten2 = new File (SparkFiles .get(file2.getName))
104+ if (! gotten1.exists()) {
105+ throw new SparkException (" file doesn't exist : " + absolutePath1)
106+ }
107+ if (! gotten2.exists()) {
108+ throw new SparkException (" file doesn't exist : " + absolutePath2)
124109 }
125- if (length != gotten.length()) {
110+
111+ if (length1 != gotten1.length()) {
126112 throw new SparkException (
127- s " file has different length $length than added file ${gotten .length()}" )
113+ s " file has different length $length1 than added file ${gotten1 .length()} : " + absolutePath1 )
128114 }
129- if (absolutePath == gotten.getAbsolutePath) {
130- throw new SparkException (" file should have been copied" )
115+ if (length2 != gotten2.length()) {
116+ throw new SparkException (
117+ s " file has different length $length2 than added file ${gotten2.length()} : " + absolutePath2)
118+ }
119+
120+ if (absolutePath1 == gotten1.getAbsolutePath) {
121+ throw new SparkException (" file should have been copied :" + absolutePath1)
122+ }
123+ if (absolutePath2 == gotten2.getAbsolutePath) {
124+ throw new SparkException (" file should have been copied : " + absolutePath2)
131125 }
132126 x
133127 }).count()
134128 } finally {
135129 sc.stop()
136130 }
137131 }
138-
132+
139133 test(" addFile recursive works" ) {
140134 val pluto = Utils .createTempDir()
141135 val neptune = Utils .createTempDir(pluto.getAbsolutePath)
0 commit comments