Update to add several fixes (seeds, updated environments) and metric collection plots (from Mark) (#30)

Update soft envs, add parameters (seeds) and incorporate Mark's plots for metric collection

Author: imallona

.github/workflows/benchmark.yml

@@ -4,10 +4,9 @@ name: micromamba
 #   cancel-in-progress: true
 
 on:
-  pull_request:
+  # pull_request:
   workflow_dispatch:
   schedule:
-    # - cron: "*/30 * * * *"  # Runs every 30 minutes for testing
     - cron: "30 1 * * *" # at 1.30am
 ## these permissions are only for deployment to gh pages
 # permissions:
@@ -22,7 +21,7 @@ jobs:
     strategy:
       matrix:
         ob_branch: [dev, reduce_install_scope, main]
-        micromamba-version: ['2.1.1-0', '2.0.5-0', '1.5.12-0', '1.5.8-0']
+        micromamba-version: ['2.1.1-0', '1.5.12-0']
       fail-fast: false
     concurrency:
       group: micromamba-${{ matrix.micromamba-version }}-${{ matrix.ob_branch }}
@@ -61,7 +60,7 @@ jobs:
         shell: bash -l {0}
         run: |
           env
-          output=$( echo "y" | ob run benchmark -b Clustering.yaml --local --cores 10 2>&1 )
+          output=$( echo "y" | ob run benchmark -b Clustering_conda.yml --local --cores 10 2>&1 )
           status=$?
           if echo "$output" | grep -i 'Benchmark run has finished successfully'; then
              status=0

.github/workflows/micromamba.yml renamed to .github/workflows/micromamba.yml.old

@@ -4,10 +4,9 @@ name: clustbench_miniforge
 #   cancel-in-progress: true
 
 on:
-  pull_request:
+  # pull_request:
   workflow_dispatch:
   schedule:
-    # - cron: "*/30 * * * *"  # Runs every 30 minutes for testing
     - cron: "30 1 * * *" # at 1.30am
 
 ## these permissions are only for deployment to gh pages
@@ -70,7 +69,7 @@ jobs:
         shell: bash -l {0}
         run: |
           env
-          output=$( echo "y" | ob run benchmark -b Clustering.yaml --local --cores 10  2>&1 )
+          output=$( echo "y" | ob run benchmark -b Clustering_conda.yml --local --cores 10  2>&1 )
           status=$?
           if echo "$output" | grep -i 'Benchmark run has finished successfully'; then
              status=0

.github/workflows/miniconda_miniforge.yml renamed to .github/workflows/miniconda_miniforge.yml.old

@@ -0,0 +1,60 @@
+name: oras
+# concurrency:
+#   group: ${{ github.head_ref || github.run_id }}
+#   cancel-in-progress: true
+
+on:
+  # pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 1 * * *" # at 1.30am
+## these permissions are only for deployment to gh pages
+# permissions:
+#   id-token: write
+#   pages: write
+
+jobs:
+  run-benchmark-oras:
+    name: run_clustbench_oras
+    runs-on: ubuntu-latest
+    # runs-on: self-hosted
+    strategy:
+      matrix:
+        ob_branch: [main]
+        micromamba-version: ['2.1.1-0']
+      fail-fast: false
+    concurrency:
+      group: oras-${{ matrix.micromamba-version }}-${{ matrix.ob_branch }}
+      cancel-in-progress: false # true
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - uses: eWaterCycle/setup-apptainer@v2
+        with:
+          apptainer-version: 1.4.0
+
+      - name: Install ob (with) micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          cache-environment: false # true
+          micromamba-version: ${{ matrix.micromamba-version }}
+          download-micromamba: true
+          micromamba-binary-path:  ${{ runner.temp }}/bin/micromamba-${{ matrix.micromamba-version }}/micromamba
+          environment-name: test-env-${{matrix.ob_branch }}-${{ matrix.micromamba-version }}
+          create-args: >-
+            python=3.12
+            pip
+            conda
+          post-cleanup: environment # all
+      
+      - name: Overwrite omnibenchmark CLI to branch
+        shell: bash -l {0}
+        run: |
+          micromamba --version
+          pip install git+https://github.com/omnibenchmark/omnibenchmark.git@${{ matrix.ob_branch }}
+
+      - name: Run benchmark
+        shell: bash -l {0}
+        run: |
+          ob run benchmark -b Clustering_oras.yml --local --cores 10 --yes

.github/workflows/oras.yml.old

@@ -0,0 +1,22 @@
+name: run_on_conda
+on:
+  push:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: "30 1 * * *" # at 1.30am
+    
+jobs:
+  run_benchmark_main:
+    name: Run on conda
+    runs-on: ubuntu-latest
+    # runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Use action 
+        uses: omnibenchmark/run_omnibenchmark@main
+        with:
+          yaml: Clustering_conda.yml
+          backend: conda
+          omnibenchmark_branch: 'dev'

.github/workflows/run_conda.yml

@@ -0,0 +1,21 @@
+name: run_on_oras
+on:
+  push:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: "30 1 * * *" # at 1.30am
+    
+jobs:
+  run-benchmark:
+    name: Run on oras
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Use action 
+        uses: omnibenchmark/run_omnibenchmark@main
+        with:
+          yaml: Clustering_oras.yml
+          backend: apptainer
+          omnibenchmark_branch: 'dev'

.github/workflows/run_oras.yml

@@ -0,0 +1,10 @@
+# image build artifacts
+envs/*.sif
+
+# snakemake
+snakemake.log
+.snakemake/
+
+# vim swaps
+*.swp
+*.swo