BlockAligner-based Smith-Waterman update

.cirrus.yml

@@ -5,10 +5,13 @@ task:
   name: FreeBSD
   freebsd_instance:
     image_family: freebsd-14-2-snap
-  install_script: pkg install -y cmake git samtools
+  install_script: |
+    # pkg update -f
+    pkg install -y cmake git samtools rust
+    export PATH=/usr/local/bin:$PATH
   compile_script: |
     mkdir build && cd build
-    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_SSE4_1=1 ..
+    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_SSE4_1=1 -DRust_COMPILER=/usr/local/bin/rustc ..
     make -j $(sysctl -n hw.ncpu)
   test_script: ./util/regression/run_regression.sh ./build/src/mmseqs SCRATCH
 
@@ -24,7 +27,9 @@ task:
         echo "deb [trusted=yes] http://archive.debian.org/debian jessie main" > /etc/apt/sources.list
         echo "deb [trusted=yes] http://archive.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list
         apt-get update --yes --force-yes || true
-        apt-get install wget clang-4.0 libc++-dev make git ca-certificates --yes --force-yes --no-install-suggests --no-install-recommends
+        apt-get install -y --no-install-suggests --no-install-recommends wget clang-4.0 libc++-dev make git ca-certificates curl
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        export PATH=/root/.cargo/bin:$PATH
       env:
         CC: clang-4.0
         CXX: clang++-4.0
@@ -33,20 +38,24 @@ task:
         echo "deb [trusted=yes] http://archive.debian.org/debian jessie main" > /etc/apt/sources.list
         echo "deb [trusted=yes] http://archive.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list
         apt-get update --yes --force-yes || true
-        apt-get install wget gcc-4.9 g++-4.9 make git ca-certificates --yes --force-yes --no-install-suggests --no-install-recommends
+        apt-get install -y --no-install-suggests --no-install-recommends wget gcc-4.9 g++-4.9 make git ca-certificates curl
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        export PATH=/root/.cargo/bin:$PATH
       env:
         CC: gcc-4.9
         CXX: g++-4.9
+
   compile_script: |
     mkdir -p /opt/cmake-3.31.0
     wget https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.sh
     chmod +x cmake-3.31.0-linux-x86_64.sh
     ./cmake-3.31.0-linux-x86_64.sh --skip-license --prefix=/opt/cmake-3.31.0
     export PATH=/opt/cmake-3.31.0/bin/:$PATH
+    export PATH=/root/.cargo/bin:$PATH
     mkdir build && cd build
-    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=0 -DHAVE_SSE4_1=1 -DREQUIRE_OPENMP=0 ..
+    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=0 -DHAVE_SSE4_1=1 -DREQUIRE_OPENMP=0 -DRust_COMPILER=/root/.cargo/bin/rustc ..
     make -j $(nproc --all)
   test_script: |
     export PATH=/opt/cmake-3.31.0/bin/:$PATH
+    export PATH=/root/.cargo/bin:$PATH
     MMSEQS_NUM_THREADS=4 ./util/regression/run_regression.sh ./build/src/mmseqs SCRATCH SEARCH
-

.github/workflows/Dockerfile.GPU-buster-cross-sbsa

@@ -7,10 +7,12 @@ RUN mkdir -p /work/build && cd /work/build; \
         -DCMAKE_TOOLCHAIN_FILE=/opt/toolchain.cmake \
         -DOpenMP_C_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_AARCH64}" -DOpenMP_C_LIB_NAMES=omp -DOpenMP_CXX_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_AARCH64}" -DOpenMP_CXX_LIB_NAMES=omp -DOpenMP_omp_LIBRARY=${LIBOMP_AARCH64}/libomp.a \
         -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_POLICY_DEFAULT_CMP0144=NEW \
+        -DRust_TOOLCHAIN=stable-x86_64-unknown-linux-gnu \
+        -DRust_CARGO_TARGET=aarch64-unknown-linux-gnu \
         -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
     cmake --build . -j$(nproc --all) -v;
 
-RUN if readelf -a /work/build/src/mmseqs | grep -i "Shared library" | grep -P -v "(linux-vdso|ld-linux-aarch64|libc|libm|libdl|librt|libpthread).so" | grep -q .; then \
+RUN if readelf -a /work/build/src/mmseqs | grep -i "Shared library" | grep -P -v "(linux-vdso|ld-linux-aarch64|libc|libm|libdl|librt|libpthread|libutil).so" | grep -q .; then \
         echo "Error: unwanted libraries found"; \
         readelf -a /work/build/src/mmseqs | grep -i "Shared library"; \
         exit 1; \

.github/workflows/Dockerfile.GPU-manylinux2014

@@ -14,7 +14,7 @@ RUN mkdir -p /work/build && cd /work/build; \
         -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
     cmake --build . -j$(nproc --all) -v;
 
-RUN if ldd /work/build/src/mmseqs | grep -P -v "linux-vdso.so|/lib64/(ld-linux-x86-64|libc|libm|libdl|librt|libpthread).so" | grep -q .; then \
+RUN if ldd /work/build/src/mmseqs | grep -P -v "linux-vdso.so|/lib64/(ld-linux-x86-64|libc|libm|libdl|librt|libpthread|libutil).so" | grep -q .; then \
         echo "Error: unwanted libraries found"; \
         ldd /work/build/src/mmseqs; \
         exit 1; \

.github/workflows/mac-arm64.yml

@@ -20,6 +20,7 @@ jobs:
       - name: Build
         run: |
           mkdir -p build
+          rustup update
           cd build
           LIBOMP=$(brew --prefix libomp)
           cmake \

CMakeLists.txt

@@ -18,6 +18,7 @@ set(HAVE_ARM8 0 CACHE BOOL "Have ARMv8 CPU")
 set(HAVE_S390X 0 CACHE BOOL "Have s390x architecture")
 set(NATIVE_ARCH 1 CACHE BOOL "Assume native architecture for SIMD. Use one of the HAVE_* options or set CMAKE_CXX_FLAGS to the appropriate flags if you disable this.")
 set(USE_SYSTEM_ZSTD 0 CACHE BOOL "Use zstd provided by system instead of bundled version")
+set(IGNORE_RUST_VERSION 0 CACHE BOOL "Ignore Rust version check")
 set(ENABLE_CUDA 0 CACHE BOOL "Enable CUDA")
 set(FORCE_STATIC_DEPS 0 CACHE BOOL "Force static linking of deps")
 
@@ -196,6 +197,53 @@ set(MMSEQS_CXX_FLAGS "${MMSEQS_CXX_FLAGS} -std=c++1y")
 #     set(MMSEQS_CXX_FLAGS "${MMSEQS_CXX_FLAGS} -stdlib=libc++")
 # endif ()
 
+set(ENV{CARGO_NET_OFFLINE} true)
+add_subdirectory(lib/corrosion)
+# don't try to link to gcc_s, its not needed anyway
+list(REMOVE_ITEM Rust_CARGO_TARGET_LINK_NATIVE_LIBS "gcc_s")
+set(Rust_CARGO_TARGET_LINK_NATIVE_LIBS ${Rust_CARGO_TARGET_LINK_NATIVE_LIBS} CACHE INTERNAL "")
+
+if(Rust_VERSION VERSION_LESS "1.68.0" AND NOT IGNORE_RUST_VERSION)
+    message(FATAL_ERROR "Rust >= 1.68.0 is required")
+endif()
+
+set(RUST_FEATURE "" CACHE INTERNAL "")
+if(HAVE_AVX2)
+    set(RUST_FEATURE simd_avx2 CACHE INTERNAL "")
+elseif(HAVE_SSE4_1 OR HAVE_SSE2)
+    set(RUST_FEATURE simd_sse2 CACHE INTERNAL "")
+elseif(HAVE_ARM8)
+    set(RUST_FEATURE simd_neon CACHE INTERNAL "")
+endif()
+
+if (NATIVE_ARCH AND (RUST_FEATURE STREQUAL ""))
+    if(EMSCRIPTEN)
+        set(RUST_FEATURE simd_wasm CACHE INTERNAL "")
+    elseif(ARM)
+        set(RUST_FEATURE simd_neon CACHE INTERNAL "")
+    elseif(X86 OR X64)
+        if (HAVE_AVX2_EXTENSIONS)
+            set(RUST_FEATURE simd_avx2 CACHE INTERNAL "")
+        elseif (HAVE_SSE2_EXTENSIONS)
+            set(RUST_FEATURE simd_sse2 CACHE INTERNAL "")
+        endif()
+    endif()
+endif()
+if(RUST_FEATURE STREQUAL "")
+    message(WARNING "No SIMD support for this architecture. Alignment performance will be bad")
+    set(RUST_FEATURE no_simd CACHE INTERNAL "")
+endif()
+message("-- Rust Feature: ${RUST_FEATURE}")
+corrosion_import_crate(
+    MANIFEST_PATH lib/block-aligner/c/Cargo.toml
+    FEATURES "${RUST_FEATURE}"
+    CRATE_TYPES staticlib
+)
+include_directories(lib/block-aligner/c)
+if(EMSCRIPTEN)
+    corrosion_add_target_local_rustflags(block_aligner_c "-Clink-args=--no-entry -sRELOCATABLE=1")
+endif()
+
 if (USE_SYSTEM_ZSTD)
     include(FindPackageHandleStandardArgs)
     find_path(ZSTD_INCLUDE_DIRS NAMES zstd.h REQUIRED)

Dockerfile

@@ -15,7 +15,9 @@ RUN dpkg --add-architecture $TARGETARCH \
       dpkg -i cuda-keyring_1.1-1_all.deb; \
       apt-get update && apt-get install -y cuda-nvcc-12-6 cuda-cudart-dev-12-6 ninja-build; \
     fi; \
-    rm -rf /var/lib/apt/lists/*;
+    rm -rf /var/lib/apt/lists/*; \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+      sh -s -- --profile minimal --default-host ${ARCH}-unknown-linux-gnu -y
 
 WORKDIR /opt/build
 ADD . .

azure-pipelines.yml

@@ -44,6 +44,11 @@ jobs:
     steps:
       - checkout: self
         submodules: true
+      - script: |
+          rustup update
+          rustup target install x86_64-apple-darwin
+          rustup target install aarch64-apple-darwin
+        displayName: Install Rust Toolchain
       - script: |
           cd ${BUILD_SOURCESDIRECTORY}
           ./util/build_osx.sh . build
@@ -89,14 +94,13 @@ jobs:
           STATIC: 0
           MPI: 0
           BUILD_TYPE: ASanOpt
-
     steps:
       - checkout: self
         submodules: true
       - script: |
           # sudo add-apt-repository ppa:ubuntu-toolchain-r/test
           sudo apt-get update
-          sudo apt-get install -y gcc-11 g++-11 libgcc-11-dev build-essential
+          sudo apt-get install -y gcc-11 g++-11 libgcc-11-dev rustc cargo build-essential
         retryCountOnTaskFailure: "2"
         displayName: Install newer G++
       - script: |
@@ -197,6 +201,7 @@ jobs:
           HEREDOC
           sudo apt-get update
           sudo apt-get -y install -o APT::Immediate-Configure=false crossbuild-essential-$ARCH zlib1g-dev:$ARCH libbz2-dev:$ARCH
+          rustup target add ${CPREF}-unknown-linux-gnu
         retryCountOnTaskFailure: "2"
         displayName: Install Toolchain
       - script: |
@@ -205,6 +210,7 @@ jobs:
           CC=${CPREF}-linux-gnu-gcc CXX=${CPREF}-linux-gnu-g++ \
            cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DHAVE_TESTS=1 -DFORCE_STATIC_DEPS=1 \
             -DOpenMP_C_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY=${LIBGOMP}/libgomp.a \
+            -DRust_CARGO_TARGET=${CPREF}-unknown-linux-gnu \
             -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 ..
           make -j $(nproc --all)
         displayName: Build MMseqs2
@@ -215,6 +221,8 @@ jobs:
 
   - job: build_windows
     displayName: Windows
+    # TODO: no rust support in cygwin yet, re-enable soon
+    condition: 'false'
     pool:
       vmImage: 'windows-2019'
     variables:
@@ -279,7 +287,7 @@ jobs:
       - build_ubuntu
       - build_ubuntu_gpu
       - build_ubuntu_cross
-      - build_windows
+      # - build_windows
     steps:
       - script: |
           cd "${BUILD_SOURCESDIRECTORY}"
@@ -405,19 +413,19 @@ jobs:
           archiveType: tar
       - script:
           rm "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
-      - task: DownloadPipelineArtifact@1
-        inputs:
-          artifactName: mmseqs-win64
-          targetPath: $(Build.SourcesDirectory)
-      - script: |
-          unzip "${BUILD_SOURCESDIRECTORY}/mmseqs-win64.zip"
-          chmod +x mmseqs/mmseqs.bat mmseqs/bin/*
-      - task: ArchiveFiles@2
-        inputs:
-          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
-          archiveFile: $(Build.SourcesDirectory)/mmseqs-win64.zip
-          includeRootFolder: true
-          archiveType: zip
+      # - task: DownloadPipelineArtifact@1
+      #   inputs:
+      #     artifactName: mmseqs-win64
+      #     targetPath: $(Build.SourcesDirectory)
+      # - script: |
+      #     unzip "${BUILD_SOURCESDIRECTORY}/mmseqs-win64.zip"
+      #     chmod +x mmseqs/mmseqs.bat mmseqs/bin/*
+      # - task: ArchiveFiles@2
+      #   inputs:
+      #     rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
+      #     archiveFile: $(Build.SourcesDirectory)/mmseqs-win64.zip
+      #     includeRootFolder: true
+      #     archiveType: zip
       - task: DownloadSecureFile@1
         inputs:
           secureFile: secretKeyPleaseDontSteal
@@ -430,6 +438,8 @@ jobs:
           cp mmseqs/userguide.pdf userguide.pdf
           # disabled: mmseqs-linux-ppc64le-power8.tar.gz
           # -F file[][email protected] -F signature[][email protected]
+          # mmseqs-win64.zip
+          # -F file[][email protected]  -F signature[][email protected]
           ssh-keygen -Y sign -f ~/.ssh/id_rsa -n file \
             userguide.pdf \
             mmseqs-osx-universal.tar.gz \
@@ -439,8 +449,7 @@ jobs:
             mmseqs-linux-gpu.tar.gz \
             mmseqs-linux-gpu-arm64.tar.gz \
             mmseqs-linux-arm64.tar.gz \
-            mmseqs-linux-ppc64le-power9.tar.gz \
-            mmseqs-win64.zip
+            mmseqs-linux-ppc64le-power9.tar.gz
           curl --fail --retry 5 --retry-all-errors -X POST \
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
@@ -451,7 +460,6 @@ jobs:
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
-            -F file[][email protected]  -F signature[][email protected] \
             -F identifier="mmseqs" -F directory="${BUILD_SOURCEVERSION}" \
               https://mmseqs.com/upload
 

lib/block-aligner/.github/workflows/ci.yaml

@@ -0,0 +1,107 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  ci-x86:
+    runs-on: ${{matrix.os}}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Rust Nightly
+      uses: actions-rs/toolchain@v1
+
+    - name: Test AVX2
+      run: |
+        cargo --version
+        cargo test --all-targets --features simd_avx2
+        cargo test --doc --features simd_avx2
+
+    - name: Test SSE2
+      run: |
+        cargo --version
+        cargo test --all-targets --features simd_sse2
+        cargo test --doc --features simd_sse2
+
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+
+    - name: Test AVX2
+      run: |
+        cargo --version
+        cargo build --features simd_avx2
+
+    - name: Test SSE2
+      run: |
+        cargo --version
+        cargo build --features simd_sse2
+
+  ci-wasm32:
+    runs-on: ${{matrix.os}}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Rust Nightly
+      uses: actions-rs/toolchain@v1
+      with:
+        target: wasm32-wasi
+
+    - name: Test WASM
+      run: |
+        WASMTIME_DIR=wasmtime-v2.0.2-x86_64-linux
+        curl -OL https://github.com/bytecodealliance/wasmtime/releases/download/v2.0.2/${WASMTIME_DIR}.tar.xz
+        tar -xvf ${WASMTIME_DIR}.tar.xz
+        mv ${WASMTIME_DIR}/wasmtime ./
+        cargo --version
+        CARGO_TARGET_WASM32_WASI_RUNNER="./wasmtime --wasm-features simd --" cargo test --target=wasm32-wasi --features simd_wasm --all-targets -- --nocapture
+
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+        target: wasm32-wasi
+
+    - name: Test WASM
+      run: |
+        cargo --version
+        cargo build --target=wasm32-wasi --features simd_wasm
+
+  ci-aarch64:
+    runs-on: ${{matrix.os}}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Rust Nightly
+      uses: actions-rs/toolchain@v1
+      with:
+        target: aarch64-unknown-linux-gnu
+
+    - name: Test Neon
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user
+        cargo --version
+        CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64 -L /usr/aarch64-linux-gnu" cargo test --target=aarch64-unknown-linux-gnu --all-targets --features simd_neon