Skip to content

Commit 38eb535

Browse files
msjgriffithsmsjgriffiths
committed
Merge pull request #2 from apache/master
Updating to latest Spark master
2 parents b08e865 + 57566d0 commit 38eb535

File tree

641 files changed

+32903
-11437
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

641 files changed

+32903
-11437
lines changed

‎.rat-excludes‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
target
2+
cache
23
.gitignore
34
.gitattributes
45
.project
@@ -18,6 +19,7 @@ fairscheduler.xml.template
1819
spark-defaults.conf.template
1920
log4j.properties
2021
log4j.properties.template
22+
metrics.properties
2123
metrics.properties.template
2224
slaves
2325
slaves.template

‎assembly/pom.xml‎

Lines changed: 10 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,6 @@
3636
<spark.jar.dir>scala-${scala.binary.version}</spark.jar.dir>
3737
<spark.jar.basename>spark-assembly-${project.version}-hadoop${hadoop.version}.jar</spark.jar.basename>
3838
<spark.jar>${project.build.directory}/${spark.jar.dir}/${spark.jar.basename}</spark.jar>
39-
<deb.pkg.name>spark</deb.pkg.name>
40-
<deb.install.path>/usr/share/spark</deb.install.path>
41-
<deb.user>root</deb.user>
42-
<deb.bin.filemode>744</deb.bin.filemode>
4339
</properties>
4440

4541
<dependencies>
@@ -118,6 +114,16 @@
118114
<exclude>META-INF/*.RSA</exclude>
119115
</excludes>
120116
</filter>
117+
<filter>
118+
<!-- Exclude libgfortran, libgcc for license issues -->
119+
<artifact>org.jblas:jblas</artifact>
120+
<excludes>
121+
<!-- Linux amd64 is OK; not statically linked -->
122+
<exclude>lib/static/Linux/i386/**</exclude>
123+
<exclude>lib/static/Mac OS X/**</exclude>
124+
<exclude>lib/static/Windows/**</exclude>
125+
</excludes>
126+
</filter>
121127
</filters>
122128
</configuration>
123129
<executions>
@@ -217,113 +223,6 @@
217223
</plugins>
218224
</build>
219225
</profile>
220-
<profile>
221-
<id>deb</id>
222-
<build>
223-
<plugins>
224-
<plugin>
225-
<groupId>org.codehaus.mojo</groupId>
226-
<artifactId>buildnumber-maven-plugin</artifactId>
227-
<version>1.2</version>
228-
<executions>
229-
<execution>
230-
<phase>validate</phase>
231-
<goals>
232-
<goal>create</goal>
233-
</goals>
234-
<configuration>
235-
<shortRevisionLength>8</shortRevisionLength>
236-
</configuration>
237-
</execution>
238-
</executions>
239-
</plugin>
240-
<plugin>
241-
<groupId>org.vafer</groupId>
242-
<artifactId>jdeb</artifactId>
243-
<version>0.11</version>
244-
<executions>
245-
<execution>
246-
<phase>package</phase>
247-
<goals>
248-
<goal>jdeb</goal>
249-
</goals>
250-
<configuration>
251-
<deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb>
252-
<attach>false</attach>
253-
<compression>gzip</compression>
254-
<dataSet>
255-
<data>
256-
<src>${spark.jar}</src>
257-
<type>file</type>
258-
<mapper>
259-
<type>perm</type>
260-
<user>${deb.user}</user>
261-
<group>${deb.user}</group>
262-
<prefix>${deb.install.path}/jars</prefix>
263-
</mapper>
264-
</data>
265-
<data>
266-
<src>${basedir}/src/deb/RELEASE</src>
267-
<type>file</type>
268-
<mapper>
269-
<type>perm</type>
270-
<user>${deb.user}</user>
271-
<group>${deb.user}</group>
272-
<prefix>${deb.install.path}</prefix>
273-
</mapper>
274-
</data>
275-
<data>
276-
<src>${basedir}/../conf</src>
277-
<type>directory</type>
278-
<mapper>
279-
<type>perm</type>
280-
<user>${deb.user}</user>
281-
<group>${deb.user}</group>
282-
<prefix>${deb.install.path}/conf</prefix>
283-
<filemode>744</filemode>
284-
</mapper>
285-
</data>
286-
<data>
287-
<src>${basedir}/../bin</src>
288-
<type>directory</type>
289-
<mapper>
290-
<type>perm</type>
291-
<user>${deb.user}</user>
292-
<group>${deb.user}</group>
293-
<prefix>${deb.install.path}/bin</prefix>
294-
<filemode>${deb.bin.filemode}</filemode>
295-
</mapper>
296-
</data>
297-
<data>
298-
<src>${basedir}/../sbin</src>
299-
<type>directory</type>
300-
<mapper>
301-
<type>perm</type>
302-
<user>${deb.user}</user>
303-
<group>${deb.user}</group>
304-
<prefix>${deb.install.path}/sbin</prefix>
305-
<filemode>744</filemode>
306-
</mapper>
307-
</data>
308-
<data>
309-
<src>${basedir}/../python</src>
310-
<type>directory</type>
311-
<mapper>
312-
<type>perm</type>
313-
<user>${deb.user}</user>
314-
<group>${deb.user}</group>
315-
<prefix>${deb.install.path}/python</prefix>
316-
<filemode>744</filemode>
317-
</mapper>
318-
</data>
319-
</dataSet>
320-
</configuration>
321-
</execution>
322-
</executions>
323-
</plugin>
324-
</plugins>
325-
</build>
326-
</profile>
327226
<profile>
328227
<id>kinesis-asl</id>
329228
<dependencies>

‎assembly/src/deb/RELEASE‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎assembly/src/deb/control/control‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎bin/compute-classpath.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ fi
7676

7777
num_jars=0
7878

79-
for f in ${assembly_folder}/spark-assembly*hadoop*.jar; do
79+
for f in "${assembly_folder}"/spark-assembly*hadoop*.jar; do
8080
if [[ ! -e "$f" ]]; then
8181
echo "Failed to find Spark assembly in $assembly_folder" 1>&2
8282
echo "You need to build Spark before running this program." 1>&2
@@ -88,7 +88,7 @@ done
8888

8989
if [ "$num_jars" -gt "1" ]; then
9090
echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2
91-
ls ${assembly_folder}/spark-assembly*hadoop*.jar 1>&2
91+
ls "${assembly_folder}"/spark-assembly*hadoop*.jar 1>&2
9292
echo "Please remove all but one jar." 1>&2
9393
exit 1
9494
fi

‎bin/run-example‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ fi
4242

4343
JAR_COUNT=0
4444

45-
for f in ${JAR_PATH}/spark-examples-*hadoop*.jar; do
45+
for f in "${JAR_PATH}"/spark-examples-*hadoop*.jar; do
4646
if [[ ! -e "$f" ]]; then
4747
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
4848
echo "You need to build Spark before running this program" 1>&2
@@ -54,7 +54,7 @@ done
5454

5555
if [ "$JAR_COUNT" -gt "1" ]; then
5656
echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2
57-
ls ${JAR_PATH}/spark-examples-*hadoop*.jar 1>&2
57+
ls "${JAR_PATH}"/spark-examples-*hadoop*.jar 1>&2
5858
echo "Please remove all but one jar." 1>&2
5959
exit 1
6060
fi

‎bin/spark-shell.cmd‎

100755100644
File mode changed.

‎bin/spark-submit2.cmd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ set ORIG_ARGS=%*
2525
rem Reset the values of all variables used
2626
set SPARK_SUBMIT_DEPLOY_MODE=client
2727

28-
if not defined %SPARK_CONF_DIR% (
28+
if [%SPARK_CONF_DIR%] == [] (
2929
set SPARK_CONF_DIR=%SPARK_HOME%\conf
3030
)
3131
set SPARK_SUBMIT_PROPERTIES_FILE=%SPARK_CONF_DIR%\spark-defaults.conf

‎bin/utils.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ function gatherSparkSubmitOpts() {
3535
--master | --deploy-mode | --class | --name | --jars | --packages | --py-files | --files | \
3636
--conf | --repositories | --properties-file | --driver-memory | --driver-java-options | \
3737
--driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
38-
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives)
38+
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives | \
39+
--proxy-user)
3940
if [[ $# -lt 2 ]]; then
4041
"$SUBMIT_USAGE_FUNCTION"
4142
exit 1;

‎bin/windows-utils.cmd‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ SET opts="%opts:~1,-1% \<--conf\> \<--properties-file\> \<--driver-memory\> \<--
3333
SET opts="%opts:~1,-1% \<--driver-library-path\> \<--driver-class-path\> \<--executor-memory\>"
3434
SET opts="%opts:~1,-1% \<--driver-cores\> \<--total-executor-cores\> \<--executor-cores\> \<--queue\>"
3535
SET opts="%opts:~1,-1% \<--num-executors\> \<--archives\> \<--packages\> \<--repositories\>"
36+
SET opts="%opts:~1,-1% \<--proxy-user\>"
3637

3738
echo %1 | findstr %opts% >nul
3839
if %ERRORLEVEL% equ 0 (

0 commit comments

Comments
 (0)