diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000000000..8b5f9d3aa4829 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,14 @@ +- project: + name: theopenlab/spark + check: + jobs: + - spark-build-and-test-x86-leveldbjni + +- job: + name: spark-build-and-test-x86-leveldbjni + parent: init-test + description: | + The spark build and test other modules in openlab cluster. + run: .zuul/playbooks/spark-build/run_all.yaml + nodeset: ubuntu-xenial + timeout: 86400 diff --git a/.zuul/playbooks/spark-build/run_all.yaml b/.zuul/playbooks/spark-build/run_all.yaml new file mode 100644 index 0000000000000..ffc8df4e2980d --- /dev/null +++ b/.zuul/playbooks/spark-build/run_all.yaml @@ -0,0 +1,37 @@ +- hosts: all + tasks: + - name: Build spark master using mvn with hadoop 2.7 + shell: + cmd: | + set -exo pipefail + sudo apt-get update -y + + # Install java + sudo apt-get install default-jre -y + sudo apt-get install default-jdk -y + java_home=$(dirname $(dirname $(update-alternatives --list javac))) + echo "export JAVA_HOME=${java_home}" >> ~/.profile + echo "export PATH=${java_home}/bin:$PATH" >> ~/.profile + source ~/.profile + + # Install maven + wget http://www.us.apache.org/dist/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz + tar -xvf apache-maven-3.6.2-bin.tar.gz + export PATH=$PWD/apache-maven-3.6.2/bin:$PATH + + # fix kafka authfail tests + sudo sed -i "s|127.0.0.1 $(hostname) localhost|127.0.0.1 localhost $(hostname)|" /etc/hosts + + cd {{ ansible_user_dir }}/{{ zuul.project.src_dir }} + + ./build/mvn clean package -DskipTests -Phadoop-2.7 -Pyarn -Phive -Phive-thriftserver -Pkinesis-asl -Pmesos + + # use leveldbjni arm supporting jar + # wget https://repo1.maven.org/maven2/org/openlabtesting/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar + # mvn install:install-file -DgroupId=org.fusesource.leveldbjni -DartifactId=leveldbjni-all -Dversion=1.8 -Dpackaging=jar -Dfile=leveldbjni-all-1.8.jar + + ./build/mvn test -Phadoop-2.7 -Pyarn -Phive -Phive-thriftserver -Pkinesis-asl -Pmesos + + chdir: '/home/zuul/src' + executable: /bin/bash + environment: '{{ global_env }}' \ No newline at end of file diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index f042a12fda3d2..a1c8a8e6582eb 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -45,7 +45,7 @@ guava - org.fusesource.leveldbjni + ${leveldbjni.group} leveldbjni-all diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 2ee17800c10e4..163c250054e4d 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -52,7 +52,7 @@ - org.fusesource.leveldbjni + ${leveldbjni.group} leveldbjni-all 1.8 diff --git a/pom.xml b/pom.xml index 44593b78c9a04..93b362ea37b1e 100644 --- a/pom.xml +++ b/pom.xml @@ -240,6 +240,7 @@ ${session.executionRootDirectory} 1g + org.fusesource.leveldbjni @@ -526,7 +527,7 @@ ${commons.httpcore.version} - org.fusesource.leveldbjni + ${leveldbjni.group} leveldbjni-all 1.8 @@ -953,6 +954,10 @@ ${hadoop.version} ${hadoop.deps.scope} + + org.fusesource.leveldbjni + leveldbjni-all + asm asm @@ -1199,6 +1204,10 @@ tests test + + org.fusesource.leveldbjni + leveldbjni-all + asm asm @@ -1239,6 +1248,10 @@ ${yarn.version} ${hadoop.deps.scope} + + org.fusesource.leveldbjni + leveldbjni-all + asm asm @@ -3081,5 +3094,17 @@ sparkr + + aarch64 + + org.openlabtesting.leveldbjni + + + + linux + aarch64 + + +