diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 3471f4abb..a373b5582 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -1,4 +1,4 @@ -name: Continuous integration tests +name: CI Tests on: push: @@ -11,16 +11,11 @@ concurrency: group: build-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - singularity_version: 3.8.3 - jobs: tox: - name: CI tests via Tox - + name: Tox runs-on: ubuntu-20.04 # 22.04 doesn't support Python 3.6 - strategy: matrix: py-ver-major: [3] @@ -42,9 +37,9 @@ jobs: - name: Set up Singularity if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} - uses: eWaterCycle/setup-singularity@v7 - with: - singularity-version: ${{ env.singularity_version }} + run: | + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -82,8 +77,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} tox-style: - name: CI linters via Tox - + name: Linters runs-on: ubuntu-20.04 strategy: @@ -118,9 +112,8 @@ jobs: run: tox conformance_tests: - name: CWL spec conformance tests - - runs-on: ubuntu-20.04 + name: CWL conformance + runs-on: ubuntu-22.04 strategy: matrix: @@ -137,9 +130,16 @@ jobs: - name: Set up Singularity if: ${{ matrix.container == 'singularity' }} - uses: eWaterCycle/setup-singularity@v7 + run: | + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + + - name: Singularity cache + if: ${{ matrix.container == 'singularity' }} + uses: actions/cache@v3 with: - singularity-version: ${{ env.singularity_version }} + path: sifcache + key: singularity - name: Set up Podman if: ${{ matrix.container == 'podman' }} @@ -148,29 +148,32 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 cache: pip - - name: Run CWL conformance tests ${{ matrix.cwl-version }} + - name: "Test CWL ${{ matrix.version }} conformance" env: - version: ${{ matrix.cwl-version }} - container: ${{ matrix.container }} - spec_branch: main + VERSION: ${{ matrix.cwl-version }} + CONTAINER: ${{ matrix.container }} + GIT_TARGET: main CWLTOOL_OPTIONS: ${{ matrix.extras }} run: ./conformance-test.sh - + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} release_test: name: cwltool release test - runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - name: Set up Singularity - uses: eWaterCycle/setup-singularity@v7 - with: - singularity-version: ${{ env.singularity_version }} + run: | + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb - name: Set up Python uses: actions/setup-python@v4 @@ -202,7 +205,7 @@ jobs: run: ./build-cwltool-docker.sh macos: - name: CI test on macos-latest + name: Test on macos-latest runs-on: macos-latest env: TOXENV: py310-unit diff --git a/conformance-test.sh b/conformance-test.sh index f180c39d3..3e773bc63 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -1,128 +1,131 @@ #!/bin/bash + venv() { - if ! test -d "$1" ; then - if command -v virtualenv > /dev/null; then - virtualenv -p python3 "$1" - else - python3 -m venv "$1" - fi + if ! test -d "$1" ; then + if command -v virtualenv > /dev/null; then + virtualenv -p python3 "$1" + else + python3 -m venv "$1" fi - # shellcheck source=/dev/null - source "$1"/bin/activate + fi + # shellcheck source=/dev/null + source "$1"/bin/activate } -# Set these environment variables when running the script, e.g.: -# version=v1.1 spec_branch=new_test container=docker ./conformance_test.sh +# Set these variables when running the script, e.g.: +# VERSION=v1.2 GIT_TARGET=1.2.1_proposed CONTAINER=podman ./conformance_test.sh # Version of the standard to test against # Current options: v1.0, v1.1, v1.2 -version=${version:-v1.0} +VERSION=${VERSION:-"v1.2"} -# Which branch of the standard's repo to use. -# This can be useful when adding new features -spec_branch=${spec_branch:-main} +# Which commit of the standard's repo to use +# Defaults to the last commit of the 1.2.1_proposed branch +GIT_TARGET=${GIT_TARGET:-"1.2.1_proposed"} # Which container runtime to use # Valid options: docker, singularity -container=${container:-docker} +CONTAINER=${CONTAINER:-docker} + +# Comma-separated list of test names that should be excluded from execution +# Defaults to "docker_entrypoint, inplace_update_on_file_content" +# EXCLUDE=${EXCLUDE:-"some_default_test_to_exclude"} set -e set -x -if [[ "$version" = "v1.0" ]] ; then - repo=common-workflow-language +# Additional arguments for the pytest command +# Defaults to none +# PYTEST_EXTRA= + +# The directory where this script resides +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Download archive from GitHub +if [[ "${VERSION}" = "v1.0" ]] ; then + REPO=common-workflow-language else # shellcheck disable=SC2001 - repo=cwl-$(echo "$version" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/') + REPO=cwl-$(echo "$VERSION" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/') fi -if [ ! -d "${repo}-${spec_branch}" ]; then - if [ ! -f "${repo}-${spec_branch}.tar.gz" ]; then - wget "https://github.com/common-workflow-language/${repo}/archive/${spec_branch}.tar.gz" - fi - tar xzf "${spec_branch}.tar.gz" +if [ ! -d "${REPO}-${GIT_TARGET}" ] ; then + if [ ! -f "${GIT_TARGET}.tar.gz" ] ; then + wget "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz" + fi + tar xzf "${GIT_TARGET}.tar.gz" fi -if [ "${container}" == "docker" ]; then +if [ "${CONTAINER}" == "docker" ]; then docker pull docker.io/node:slim fi -if [ "${container}" == "podman" ]; then +if [ "${CONTAINER}" == "podman" ]; then podman pull docker.io/node:slim fi -venv cwltool-venv3 -pip3 install -U setuptools wheel pip -pip3 uninstall -y cwltool -pip3 install -e . -pip3 install codecov cwltest>=2.1 -root_folder=${PWD} -pushd "${repo}-${spec_branch}" || exit 1 - -# shellcheck disable=SC2043 -if [[ "$version" = "v1.0" ]]; then - DRAFT="DRAFT=v1.0" +if [ "${CONTAINER}" == "singularity" ]; then + export CWL_SINGULARITY_CACHE="$SCRIPT_DIRECTORY/sifcache" + mkdir --parents "${CWL_SINGULARITY_CACHE}" fi -# Clean up all cov data -find . -name '.coverage*' -print0 | xargs -0 rm -f -rm -f coverage.xml - -COVERAGE_RC=${PWD}/.coveragerc -cat > "${COVERAGE_RC}" < "${CWLTOOL_WITH_COV}" <=2.3 pytest-cov pytest-xdist + +# Set conformance test filename +if [[ "${VERSION}" = "v1.0" ]] ; then + CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml" +else + CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/conformance_tests.yaml" +fi +cp "${CONFORMANCE_TEST}" "${CONFORMANCE_TEST%".yaml"}.cwltest.yaml" +CONFORMANCE_TEST="${CONFORMANCE_TEST%".yaml"}.cwltest.yaml" CWLTOOL_OPTIONS+=" --parallel" -# shellcheck disable=SC2154 -if [[ "$version" = *dev* ]] +unset exclusions +declare -a exclusions +if [[ "$VERSION" = *dev* ]] then CWLTOOL_OPTIONS+=" --enable-dev" fi - -if [[ "$container" = "singularity" ]]; then +if [[ "$CONTAINER" = "singularity" ]]; then CWLTOOL_OPTIONS+=" --singularity" # This test fails because Singularity and Docker have # different views on how to deal with this. exclusions+=(docker_entrypoint) - - if [[ "${version}" = "v1.1" ]]; then + if [[ "${VERSION}" = "v1.1" ]]; then # This fails because of a difference (in Singularity vs Docker) in # the way filehandles are passed to processes in the container and # wc can tell somehow. # See issue #1440 exclusions+=(stdin_shorcut) fi -elif [[ "$container" = "podman" ]]; then +elif [[ "$CONTAINER" = "podman" ]]; then CWLTOOL_OPTIONS+=" --podman" fi -if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] && [[ "$container" = "docker" ]] +if [[ -n "${EXCLUDE}" ]] ; then + EXCLUDE="${EXCLUDE}," +fi +if (( "${#exclusions[*]}" > 0 )); then + EXCLUDE=${EXCLUDE}$(IFS=,; echo "${exclusions[*]}") +fi + +# Build command +TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${SCRIPT_DIRECTORY}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)" +if [[ -n "${EXCLUDE}" ]] ; then + TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}" +fi +TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}" + +# Clean up all old coverage data +find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete + +if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$VERSION" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]] then rm -Rf conformance # shellcheck disable=SC2154 @@ -130,59 +133,47 @@ then git -C conformance config user.email "cwl-bot@users.noreply.github.com" git -C conformance config user.name "CWL Jenkins build bot" + tool_ver=$(cwltool --version | awk '{ print $2 }') + badgedir=${PWD}/conformance/cwltool/cwl_${VERSION}/cwltool_${tool_ver} + mkdir -p "${PWD}"/conformance/cwltool/cwl_"${VERSION}"/ + rm -fr "${badgedir}" + TEST_COMMAND="${TEST_COMMAND} --cwl-badgedir=${badgedir}" CONFORMANCE_MSG=$(cat << EOM -Conformance test of cwltool ${tool_ver} for CWL ${version} +Conformance test of cwltool ${tool_ver} for CWL ${VERSION} Commit: ${GIT_COMMIT} Python version: 3 -Container: ${container} +Container: ${CONTAINER} Extra options: ${CWLTOOL_OPTIONS} EOM ) - - tool_ver=$(cwltool --version | awk '{ print $2 }') - badgedir=${PWD}/conformance/cwltool/cwl_${version}/cwltool_${tool_ver} - mkdir -p "${PWD}"/conformance/cwltool/cwl_"${version}"/ - rm -fr "${badgedir}" - BADGE=" --badgedir=${badgedir}" fi -if (( "${#exclusions[*]}" > 0 )); then - EXCLUDE=-S$(IFS=,; echo "${exclusions[*]}") -else - EXCLUDE="" -fi export CWLTOOL_OPTIONS echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}" -# shellcheck disable=SC2086 -LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml ${EXCLUDE} \ - RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \ - ${DRAFT} \ - "--classname=py3_${container}_$(echo ${CWLTOOL_OPTIONS} | tr "[:blank:]-" _)" -# LC_ALL=C is to work around junit-xml ASCII only bug -# capture return code of ./run_test.sh -CODE=$? +# Run test +cp "${SCRIPT_DIRECTORY}/tests/cwl-conformance/cwltool-conftest.py" "$(dirname "${CONFORMANCE_TEST}")/conftest.py" +bash -c "${TEST_COMMAND}" +RETURN_CODE=$? -find . -name '.coverage.*' -print0 | xargs -0 coverage combine --rcfile="${COVERAGE_RC}" --append -coverage xml --rcfile="${COVERAGE_RC}" -codecov --file coverage.xml +# Coverage report +if [ "${RETURN_CODE}" -eq "0" ] ; then + coverage report + coverage xml +fi if [ -d conformance ] then - rm -rf conformance/cwltool/cwl_"${version}"/cwltool_latest - cp -r conformance/cwltool/cwl_"${version}"/cwltool_"${tool_ver}" conformance/cwltool/cwl_"${version}"/cwltool_latest + rm -rf conformance/cwltool/cwl_"${VERSION}"/cwltool_latest + cp -r conformance/cwltool/cwl_"${VERSION}"/cwltool_"${tool_ver}" conformance/cwltool/cwl_"${VERSION}"/cwltool_latest git -C conformance add --all git -C conformance diff-index --quiet HEAD || git -C conformance commit -m "${CONFORMANCE_MSG}" git -C conformance push http://"${jenkins_cwl_conformance}":x-oauth-basic@github.com/common-workflow-language/conformance.git fi -popd || exit +# Cleanup deactivate +#rm -rf "${GIT_TARGET}.tar.gz" "${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}" "${SCRIPT_DIRECTORY}/cwl-conformance-venv" -# build new docker container -# if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] -# then -# ./build-cwltool-docker.sh || true -# fi -#docker rm -v $(docker ps -a -f status=exited | sed 's/ */ /g' | cut -d' ' -f1) -exit ${CODE} +# Exit +exit ${RETURN_CODE} diff --git a/release-test.sh b/release-test.sh index 2ef2fda50..663b552c3 100755 --- a/release-test.sh +++ b/release-test.sh @@ -43,14 +43,14 @@ then && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel pip install -rtest-requirements.txt ".${extras}" - make test + #make test pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install - mkdir testenv1/not-${module} + # mkdir testenv1/not-${module} # if there is a subdir named '${module}' py.test will execute tests # there instead of the installed module's tests - pushd testenv1/not-${module} - # shellcheck disable=SC2086 - test_prefix=../ run_tests; popd + # pushd testenv1/not-${module} + # #shellcheck disable=SC2086 + # test_prefix=../ run_tests; popd fi python3 -m venv testenv2 @@ -71,12 +71,12 @@ pip install -e "git+${repo}@${HEAD}#egg=${package}${extras}" pushd src/${package} pip install -rtest-requirements.txt make dist -make test +#make test cp dist/${package}*tar.gz ../../../testenv3/ pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install popd # ../.. no subdir named ${proj} here, safe for py.testing the installed module # shellcheck disable=SC2086 -run_tests +#run_tests popd # Is the source distribution in testenv2 complete enough to build another diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py new file mode 100644 index 000000000..e846b2706 --- /dev/null +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -0,0 +1,35 @@ +""" +Example configuration for pytest + cwltest plugin using cwltool directly. + +Calls cwltool via Python, instead of a subprocess via `--cwl-runner cwltool`. +""" +import json +from io import StringIO +from typing import Any, Dict, List, Optional, Tuple + +from cwltest import utils + + +def pytest_cwl_execute_test( + config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] +) -> Tuple[int, Optional[Dict[str, Any]]]: + """Use the CWL reference runner (cwltool) to execute tests.""" + from cwltool import main + from cwltool.errors import WorkflowException + + stdout = StringIO() + argsl: List[str] = [f"--outdir={config.outdir}"] + if config.runner_quiet: + argsl.append("--quiet") + elif config.verbose: + argsl.append("--debug") + argsl.extend(config.args) + argsl.append(processfile) + if jobfile: + argsl.append(jobfile) + try: + result = main.main(argsl=argsl, stdout=stdout) + except WorkflowException: + return 1, {} + out = stdout.getvalue() + return result, json.loads(out) if out else {}