From 7c4c6be5f1dc69923f1063b65d76bbd2f106261c Mon Sep 17 00:00:00 2001 From: tellet-q Date: Wed, 9 Apr 2025 18:38:55 +0200 Subject: [PATCH 1/2] Run benchmarks with QRANT__FEATURE_FLAGS__ALL=true for dev only --- .github/workflows/continuous-benchmark-2.yaml | 2 ++ .github/workflows/continuous-benchmark.yaml | 6 ++++++ .../qdrant-continuous-benchmarks/docker-compose.yaml | 2 ++ tools/run_server_container.sh | 3 ++- tools/run_server_container_with_volume.sh | 5 +++-- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continuous-benchmark-2.yaml b/.github/workflows/continuous-benchmark-2.yaml index 1ae63e31..d7f9dcae 100644 --- a/.github/workflows/continuous-benchmark-2.yaml +++ b/.github/workflows/continuous-benchmark-2.yaml @@ -58,10 +58,12 @@ jobs: # Benchmark the dev branch: export QDRANT_VERSION=ghcr/dev + export QDRANT__FEATURE_FLAGS__ALL=true timeout 30m bash -x tools/run_ci.sh # Benchmark the master branch: export QDRANT_VERSION=docker/master + export QDRANT__FEATURE_FLAGS__ALL=false timeout 30m bash -x tools/run_ci.sh done diff --git a/.github/workflows/continuous-benchmark.yaml b/.github/workflows/continuous-benchmark.yaml index 07a5f2cf..3717ae8f 100644 --- a/.github/workflows/continuous-benchmark.yaml +++ b/.github/workflows/continuous-benchmark.yaml @@ -50,10 +50,12 @@ jobs: # Benchmark the dev branch: export QDRANT_VERSION=ghcr/dev + export QDRANT__FEATURE_FLAGS__ALL=true timeout 30m bash -x tools/run_ci.sh # Benchmark the master branch: export QDRANT_VERSION=docker/master + export QDRANT__FEATURE_FLAGS__ALL=false timeout 30m bash -x tools/run_ci.sh done @@ -140,10 +142,12 @@ jobs: # Benchmark the dev branch: export QDRANT_VERSION=ghcr/dev + export QDRANT__FEATURE_FLAGS__ALL=true timeout 30m bash -x tools/run_ci.sh # Benchmark the master branch: export QDRANT_VERSION=docker/master + export QDRANT__FEATURE_FLAGS__ALL=false timeout 30m bash -x tools/run_ci.sh set -e @@ -229,10 +233,12 @@ jobs: # Benchmark the dev branch: export QDRANT_VERSION=ghcr/dev + export QDRANT__FEATURE_FLAGS__ALL=true timeout 30m bash -x tools/run_ci.sh # Benchmark the master branch: export QDRANT_VERSION=docker/master + export QDRANT__FEATURE_FLAGS__ALL=false timeout 30m bash -x tools/run_ci.sh set -e diff --git a/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml b/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml index ada808d6..83a5c900 100644 --- a/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml +++ b/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml @@ -16,3 +16,5 @@ services: resources: limits: memory: 25Gb + environment: + - QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-false} diff --git a/tools/run_server_container.sh b/tools/run_server_container.sh index 79648605..249ad92a 100644 --- a/tools/run_server_container.sh +++ b/tools/run_server_container.sh @@ -16,6 +16,7 @@ SCRIPT_PATH=$(dirname "$SCRIPT") BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"} QDRANT_VERSION=${QDRANT_VERSION:-"dev"} +QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-"false"} IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME") @@ -34,7 +35,7 @@ if [[ ${QDRANT_VERSION} == docker/* ]] || [[ ${QDRANT_VERSION} == ghcr/* ]]; the CONTAINER_REGISTRY='ghcr.io' fi - DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d; docker container ls -a" + DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d; docker container ls -a" ssh -t -o ServerAliveInterval=60 -o ServerAliveCountMax=3 "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/vector-db-benchmark/engine/servers/${CONTAINER_NAME} ; $DOCKER_COMPOSE" else echo "Error: unknown version ${QDRANT_VERSION}. Version name should start with 'docker/' or 'ghcr/'" diff --git a/tools/run_server_container_with_volume.sh b/tools/run_server_container_with_volume.sh index 294db760..fb1f3ade 100644 --- a/tools/run_server_container_with_volume.sh +++ b/tools/run_server_container_with_volume.sh @@ -17,6 +17,7 @@ SCRIPT_PATH=$(dirname "$SCRIPT") BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"} QDRANT_VERSION=${QDRANT_VERSION:-"dev"} +QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-"false"} IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME") @@ -37,11 +38,11 @@ if [[ ${QDRANT_VERSION} == docker/* ]] || [[ ${QDRANT_VERSION} == ghcr/* ]]; the if [[ "$EXECUTION_MODE" == "init" ]]; then echo "Initialize qdrant from scratch, with qdrant_storage volume" - DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true; docker volume rm -f qdrant_storage || true; docker compose up -d; docker container ls -a" + DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; export QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true; docker volume rm -f qdrant_storage || true; docker compose up -d; docker container ls -a" elif [[ "$EXECUTION_MODE" == "continue" ]]; then # suggest that volume qdrant_storage exist and start qdrant echo "Reload qdrant with existing data" - DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; sudo bash -c 'sync; echo 1 > /proc/sys/vm/drop_caches'; docker compose up -d; docker container ls -a" + DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; export QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; sudo bash -c 'sync; echo 1 > /proc/sys/vm/drop_caches'; docker compose up -d; docker container ls -a" else echo "Error: unknown execution mode ${EXECUTION_MODE}. Execution mode should be 'init' or 'continue'" exit 1 From b6e080b6c9684494533c31c817a39a8dc01f18b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Apr 2025 16:40:25 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/continuous-benchmark.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yaml b/.github/workflows/continuous-benchmark.yaml index 3717ae8f..0056b6ab 100644 --- a/.github/workflows/continuous-benchmark.yaml +++ b/.github/workflows/continuous-benchmark.yaml @@ -142,12 +142,12 @@ jobs: # Benchmark the dev branch: export QDRANT_VERSION=ghcr/dev - export QDRANT__FEATURE_FLAGS__ALL=true + export QDRANT__FEATURE_FLAGS__ALL=true timeout 30m bash -x tools/run_ci.sh # Benchmark the master branch: export QDRANT_VERSION=docker/master - export QDRANT__FEATURE_FLAGS__ALL=false + export QDRANT__FEATURE_FLAGS__ALL=false timeout 30m bash -x tools/run_ci.sh set -e