From 9d074b0de379155cd24f6326c9cda8f15d8c4a48 Mon Sep 17 00:00:00 2001 From: Foivos Zakkak Date: Wed, 20 Sep 2023 10:56:37 +0300 Subject: [PATCH 1/3] [CI] New build type that allows testing without maven-deploy-local --- .github/workflows/base-wrapper.yml | 4 +- .github/workflows/base.yml | 85 ++++++++++++++++++++---------- 2 files changed, 59 insertions(+), 30 deletions(-) diff --git a/.github/workflows/base-wrapper.yml b/.github/workflows/base-wrapper.yml index 1f845be74b97..ff4efc4a15c7 100644 --- a/.github/workflows/base-wrapper.yml +++ b/.github/workflows/base-wrapper.yml @@ -35,11 +35,13 @@ on: default: "graalvm/mandrel-packaging" build-type: type: choice - description: 'Build distribution (graal/mandrel) from source or use released binaries' + description: 'Build distribution (graal/mandrel) from source or use released binaries, and control of maven should deploy locally' default: "mandrel-source" options: - "mandrel-source" - "graal-source" + - "mandrel-source-nolocalmvn" + - "graal-source-nolocalmvn" - "mandrel-release" - "graal-release" jdk: diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 76c90b52108a..3bfd4e9134ed 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -30,7 +30,7 @@ on: default: "graalvm/mandrel-packaging" build-type: type: string - description: 'Build distribution (Mandrel/GraalVM) from source or grab a release' + description: 'Build distribution (Mandrel/GraalVM) from source or grab a release, and control of maven should deploy locally' default: "mandrel-source" jdk: type: string @@ -85,11 +85,12 @@ env: jobs: build-vars: - name: Set distribution and build-from-source variables based on build-type + name: Set distribution, build-from-source, and maven-deploy-local variables based on build-type runs-on: ubuntu-latest outputs: build-from-source: ${{ steps.source-build.outputs.build-from-source }} distribution: ${{ steps.distro.outputs.distribution }} + maven-deploy-local: ${{ steps.maven-deploy-local.outputs.maven-deploy-local }} steps: - name: Set build-from-source output id: source-build @@ -124,6 +125,18 @@ jobs: fi echo "distro=$distro" echo "distribution=$distro" >> $GITHUB_OUTPUT + - name: Set maven-deploy-local output + id: maven-deploy-local + run: | + bfs_token=$(echo ${{ inputs.build-type }} | cut -d'-' -f3) + if [ "${bfs_token}" == "nolocalmvn" ] + then + maven_deploy_local="" + else + maven_deploy_local="--maven-deploy-local" + fi + echo "maven_deploy_local=$maven_deploy_local" + echo "maven-deploy-local=$maven_deploy_local" >> $GITHUB_OUTPUT get-test-matrix: name: Get test matrix @@ -210,20 +223,27 @@ jobs: - name: Build Mandrel if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'mandrel' && inputs.builder-image == 'null'}} run: | - rm -rf ~/.m2/repository/org/graalvm + MVN_LOCAL="${{needs.build-vars.outputs.maven-deploy-local}}" + if [ "$MVN_LOCAL" != "" ] + then + rm -rf ~/.m2/repository/org/graalvm + fi cd ${MANDREL_PACKAGING_REPO} ${JAVA_HOME}/bin/java -ea build.java \ --mx-home ${MX_PATH} \ --mandrel-repo ${MANDREL_REPO} \ --mandrel-home ${MANDREL_HOME} \ --archive-suffix tar.gz \ - --maven-deploy-local + "${{needs.build-vars.outputs.maven-deploy-local}}" ${MANDREL_HOME}/bin/native-image --version mv mandrel-java*-linux-amd64-*.tar.gz ${{ github.workspace }}/jdk.tgz - rm -rf graalvm-maven-artefacts.tgz && tar -czvf graalvm-maven-artefacts.tgz -C ~ .m2/repository/org/graalvm - mv graalvm-maven-artefacts.tgz ${{ github.workspace }} - rm -rf graalvm-version.tgz && tar -czvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME}) $(basename ${MANDREL_HOME})/.maven-version - mv graalvm-version.tgz ${{ github.workspace }} + if [ "$MVN_LOCAL" != "" ] + then + rm -rf graalvm-maven-artefacts.tgz && tar -czvf graalvm-maven-artefacts.tgz -C ~ .m2/repository/org/graalvm + mv graalvm-maven-artefacts.tgz ${{ github.workspace }} + rm -rf graalvm-version.tgz && tar -czvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME}) $(basename ${MANDREL_HOME})/.maven-version + mv graalvm-version.tgz ${{ github.workspace }} + fi - name: Persist Mandrel build if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'mandrel' && inputs.builder-image == 'null'}} uses: actions/upload-artifact@v3 @@ -231,13 +251,13 @@ jobs: name: jdk-${{ needs.get-test-matrix.outputs.artifacts-suffix }} path: jdk.tgz - name: Persist local maven repository - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'mandrel' && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'mandrel' && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} uses: actions/upload-artifact@v3 with: name: org-graalvm-artefacts-${{ needs.get-test-matrix.outputs.artifacts-suffix }} path: graalvm-maven-artefacts.tgz - name: Persist GraalVM maven version - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'mandrel' && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'mandrel' && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} uses: actions/upload-artifact@v3 with: name: mandrel-maven-version-${{ needs.get-test-matrix.outputs.artifacts-suffix }} @@ -283,21 +303,28 @@ jobs: - name: Build graalvm native-image if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'graalvm' && inputs.builder-image == 'null'}} run: | - rm -rf ~/.m2/repository/org/graalvm + MVN_LOCAL="${{needs.build-vars.outputs.maven-deploy-local}}" + if [ "$MVN_LOCAL" != "" ] + then + rm -rf ~/.m2/repository/org/graalvm + fi cd graal/substratevm ${MX_PATH}/mx --native=native-image,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent --components="Native Image,LibGraal,svml" build mv $(${MX_PATH}/mx --native=native-image,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent --components="Native Image,LibGraal,svml" graalvm-home) ${MANDREL_HOME} ${MANDREL_HOME}/bin/native-image --version - cd ../ - # Deploy maven artefacts to local repository - ${MX_PATH}/mx --primary-suite-path sdk maven-deploy --suppress-javadoc 2>&1 | tee maven_deploy.log - rm -rf graalvm-maven-artefacts.tgz && tar -czvf graalvm-maven-artefacts.tgz -C ~ .m2/repository/org/graalvm - mv graalvm-maven-artefacts.tgz ${{ github.workspace }} - MAVEN_VERS=$(for token in $(head -n1 maven_deploy.log); do if echo $token | grep -q 'SNAPSHOT'; then echo $token; fi; done) - echo "GraalVM maven version is: ${MAVEN_VERS}" - echo ${MAVEN_VERS} > ${MANDREL_HOME}/.maven-version - rm -rf graalvm-version.tgz && tar -czvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME}) $(basename ${MANDREL_HOME})/.maven-version - mv graalvm-version.tgz ${{ github.workspace }} + if [ "$MVN_LOCAL" != "" ] + then + cd ../ + # Deploy maven artefacts to local repository + ${MX_PATH}/mx --primary-suite-path sdk maven-deploy --suppress-javadoc 2>&1 | tee maven_deploy.log + rm -rf graalvm-maven-artefacts.tgz && tar -czvf graalvm-maven-artefacts.tgz -C ~ .m2/repository/org/graalvm + mv graalvm-maven-artefacts.tgz ${{ github.workspace }} + MAVEN_VERS=$(for token in $(head -n1 maven_deploy.log); do if echo $token | grep -q 'SNAPSHOT'; then echo $token; fi; done) + echo "GraalVM maven version is: ${MAVEN_VERS}" + echo ${MAVEN_VERS} > ${MANDREL_HOME}/.maven-version + rm -rf graalvm-version.tgz && tar -czvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME}) $(basename ${MANDREL_HOME})/.maven-version + mv graalvm-version.tgz ${{ github.workspace }} + fi - name: Tar GraalVM if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'graalvm' && inputs.builder-image == 'null'}} run: tar czvf jdk.tgz -C $(dirname ${MANDREL_HOME}) $(basename ${MANDREL_HOME}) @@ -308,13 +335,13 @@ jobs: name: jdk-${{ needs.get-test-matrix.outputs.artifacts-suffix }} path: jdk.tgz - name: Persist local maven repository - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'graalvm' && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'graalvm' && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} uses: actions/upload-artifact@v3 with: name: org-graalvm-artefacts-${{ needs.get-test-matrix.outputs.artifacts-suffix }} path: graalvm-maven-artefacts.tgz - name: Persist GraalVM maven version - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'graalvm' && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && needs.build-vars.outputs.distribution == 'graalvm' && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} uses: actions/upload-artifact@v3 with: name: mandrel-maven-version-${{ needs.get-test-matrix.outputs.artifacts-suffix }} @@ -394,25 +421,25 @@ jobs: distribution: 'temurin' java-version: '11' - name: Download GraalVM Maven Repo - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} uses: actions/download-artifact@v3 with: name: org-graalvm-artefacts-${{ needs.get-test-matrix.outputs.artifacts-suffix }} path: . - name: Download GraalVM Maven Version - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} uses: actions/download-artifact@v3 with: name: mandrel-maven-version-${{ needs.get-test-matrix.outputs.artifacts-suffix }} path: . - name: Extract GraalVM Maven Repo and GraalVM Maven Version - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} shell: bash run: | tar -xzvf graalvm-maven-artefacts.tgz -C ~ tar -xzvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME}) - name: Build quarkus with local graalvm version - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} run: | rm -f maven_graalvm_before_build.txt maven_graalvm_after_build.txt find ~/.m2/repository/org/graalvm | sort > maven_graalvm_before_build.txt @@ -424,7 +451,7 @@ jobs: find ~/.m2/repository/org/graalvm | sort > maven_graalvm_after_build.txt diff -u maven_graalvm_before_build.txt maven_graalvm_after_build.txt - name: Build quarkus with default graalvm version - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == false || inputs.builder-image != 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == false || inputs.builder-image != 'null' || needs.build-vars.outputs.maven-deploy-local == ''}} run: | cd ${QUARKUS_PATH} ./mvnw -e -B --settings ${QUARKUS_PATH}/.github/mvn-settings.xml -Dquickly @@ -443,7 +470,7 @@ jobs: run: | rm -r ~/.m2/repository/io/quarkus - name: Delete Local GraalVM Artifacts From Cache - if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null'}} + if: ${{ fromJson(needs.build-vars.outputs.build-from-source) == true && inputs.builder-image == 'null' && needs.build-vars.outputs.maven-deploy-local != ''}} shell: bash run: | rm -rf ~/.m2/repository/org/graalvm From e2511383ffde584e4798c247b2ed596ee26f647f Mon Sep 17 00:00:00 2001 From: Foivos Zakkak Date: Wed, 20 Sep 2023 10:57:15 +0300 Subject: [PATCH 2/3] [CI] Add JDK 22 option --- .github/workflows/base-wrapper.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/base-wrapper.yml b/.github/workflows/base-wrapper.yml index ff4efc4a15c7..0b3c55d9e534 100644 --- a/.github/workflows/base-wrapper.yml +++ b/.github/workflows/base-wrapper.yml @@ -56,6 +56,8 @@ on: - "20/ga" - "21/ga" - "21/ea" + - "22/ga" + - "22/ea" builder-image: type: string description: 'The builder image to use instead of a release or building from source (e.g. quay.io/quarkus/ubi-quarkus-mandrel:20.3-java11)' From 8c840b3cf90ca3127d2b84f6c57953167bd83cf8 Mon Sep 17 00:00:00 2001 From: Foivos Zakkak Date: Wed, 20 Sep 2023 13:49:06 +0300 Subject: [PATCH 3/3] [CI] Don't assume SNAPSHOT suffix in version of local artifacts --- .github/workflows/base.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 3bfd4e9134ed..9a822596d45b 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -319,7 +319,8 @@ jobs: ${MX_PATH}/mx --primary-suite-path sdk maven-deploy --suppress-javadoc 2>&1 | tee maven_deploy.log rm -rf graalvm-maven-artefacts.tgz && tar -czvf graalvm-maven-artefacts.tgz -C ~ .m2/repository/org/graalvm mv graalvm-maven-artefacts.tgz ${{ github.workspace }} - MAVEN_VERS=$(for token in $(head -n1 maven_deploy.log); do if echo $token | grep -q 'SNAPSHOT'; then echo $token; fi; done) + # Get version from first line of the log, which should look like "Installing sdk distributions for version 23.1.0.0" + MAVEN_VERS=$(awk 'NR==1 {print $6}' maven_deploy.log) echo "GraalVM maven version is: ${MAVEN_VERS}" echo ${MAVEN_VERS} > ${MANDREL_HOME}/.maven-version rm -rf graalvm-version.tgz && tar -czvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME}) $(basename ${MANDREL_HOME})/.maven-version