diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 0000000000000..74308e11f433f
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,14 @@
+- project:
+ name: theopenlab/spark
+ check:
+ jobs:
+ - spark-build-and-python-test-arm64
+
+- job:
+ name: spark-build-and-python-test-arm64
+ parent: init-test
+ description: |
+ The spark build and test other modules in openlab cluster.
+ run: .zuul/playbooks/spark-build/run_python_tests.yaml
+ nodeset: ubuntu-xenial-arm64
+ timeout: 86400
diff --git a/.zuul/playbooks/spark-build/run_python_tests.yaml b/.zuul/playbooks/spark-build/run_python_tests.yaml
new file mode 100644
index 0000000000000..b446294ff698c
--- /dev/null
+++ b/.zuul/playbooks/spark-build/run_python_tests.yaml
@@ -0,0 +1,53 @@
+- hosts: all
+ tasks:
+ - name: Build spark master using mvn with hadoop 2.7
+ shell:
+ cmd: |
+ set -exo pipefail
+ sudo apt-get update -y
+
+ # Install java
+ sudo apt-get install default-jre -y
+ sudo apt-get install default-jdk -y
+ java_home=$(dirname $(dirname $(update-alternatives --list javac)))
+ echo "export JAVA_HOME=${java_home}" >> ~/.profile
+ echo "export PATH=${java_home}/bin:$PATH" >> ~/.profile
+ source ~/.profile
+
+ # Install maven
+ wget http://www.us.apache.org/dist/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz
+ tar -xvf apache-maven-3.6.2-bin.tar.gz
+ export PATH=$PWD/apache-maven-3.6.2/bin:$PATH
+
+ # fix kafka authfail tests
+ sudo sed -i "s|127.0.0.1 $(hostname) localhost|127.0.0.1 localhost $(hostname)|" /etc/hosts
+
+ cd {{ ansible_user_dir }}/{{ zuul.project.src_dir }}
+
+ ./build/mvn install -DskipTests -Phadoop-2.7 -Pyarn -Phive -Phive-thriftserver -Pkinesis-asl -Pmesos
+
+ # use leveldbjni arm supporting jar
+ wget https://repo1.maven.org/maven2/org/openlabtesting/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar
+ mvn install:install-file -DgroupId=org.fusesource.leveldbjni -DartifactId=leveldbjni-all -Dversion=1.8 -Dpackaging=jar -Dfile=leveldbjni-all-1.8.jar
+
+ # install python3.6
+ sudo add-apt-repository ppa:jonathonf/python-3.6 -y
+ sudo apt-get update -y
+ sudo apt-get install python3.6 -y
+ sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+ sudo apt-get install python3.6-dev -y
+
+ # install pip(pip3)
+ curl https://bootstrap.pypa.io/get-pip.py | sudo python3.6
+
+ # install packages needed
+ sudo pip2 install coverage numpy
+ sudo pip install coverage numpy
+
+ sleep 36000
+ # run python tests
+ python/run-tests --python-executables=python2.7,python3.6
+
+ chdir: '/home/zuul/src'
+ executable: /bin/bash
+ environment: '{{ global_env }}'
\ No newline at end of file
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index f042a12fda3d2..705061620f107 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -45,8 +45,9 @@
guava
- org.fusesource.leveldbjni
+ ${leveldbjni.group}
leveldbjni-all
+ 1.8
com.fasterxml.jackson.core
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 2ee17800c10e4..163c250054e4d 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -52,7 +52,7 @@
- org.fusesource.leveldbjni
+ ${leveldbjni.group}
leveldbjni-all
1.8
diff --git a/pom.xml b/pom.xml
index bc8595a1f4ef7..6d7262bc9c22f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -241,6 +241,7 @@
${session.executionRootDirectory}
1g
+ org.fusesource.leveldbjni
@@ -527,7 +528,7 @@
${commons.httpcore.version}
- org.fusesource.leveldbjni
+ ${leveldbjni.group}
leveldbjni-all
1.8
@@ -3073,5 +3074,11 @@
sparkr
+
diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 6f098f4582930..97197522c8a9d 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -33,14 +33,14 @@
class MLLibStreamingTestCase(unittest.TestCase):
def setUp(self):
self.sc = SparkContext('local[4]', "MLlib tests")
- self.ssc = StreamingContext(self.sc, 1.0)
+ self.ssc = StreamingContext(self.sc, 3.0)
def tearDown(self):
self.ssc.stop(False)
self.sc.stop()
@staticmethod
- def _eventually(condition, timeout=30.0, catch_assertions=False):
+ def _eventually(condition, timeout=120.0, catch_assertions=False):
"""
Wait a given amount of time for a condition to pass, else fail with an error.
This is a helper utility for streaming ML tests.
@@ -289,7 +289,7 @@ def condition():
return True
# We want all batches to finish for this test.
- self._eventually(condition, 60.0, catch_assertions=True)
+ self._eventually(condition, catch_assertions=True)
t_models = array(models)
diff = t_models[1:] - t_models[:-1]
@@ -364,7 +364,7 @@ def condition():
return True
return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
- self._eventually(condition, timeout=60.0)
+ self._eventually(condition)
class StreamingLinearRegressionWithTests(MLLibStreamingTestCase):