Add Warp-specific IBM Cloud VM nightly runs tooling

Signed-off-by: Ben <[email protected]>

Author: Neon-White

.github/ibm-warp-runner-config.yaml

@@ -0,0 +1,38 @@
+#cloud-config
+package_update: true
+packages: [docker.io, git, make, unzip, jq, curl, ca-certificates, nodejs, npm]
+
+write_files:
+  - path: /etc/warp.env
+    permissions: '0600'
+    content: |
+      NODE_PATH=/usr/local/lib/node_modules
+      SLACK_NIGHTLY_RESULTS_URL=${SLACK_NIGHTLY_RESULTS_URL}
+      AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      WARP_LOGS_BUCKET=${WARP_LOGS_BUCKET}
+      IBM_COS_ENDPOINT=${IBM_COS_ENDPOINT}
+
+runcmd:
+  # Schedule VM shutdown after 4 hours (240 minutes) as a safety measure
+  - [ shutdown, -P, '+240' ]
+  # Add ubuntu user to docker group for container access
+  - [ usermod, -aG, docker, ubuntu ]
+  # Enable and start Docker daemon
+  - [ systemctl, enable, --now, docker ]
+  # Install Slack webhook package globally for the notifier script
+  - [ npm, -g, install, '@slack/webhook' ]
+  # Download, extract and install AWS CLI v2
+  - [ curl, -sS, https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip, -o, awscliv2.zip ]
+  - [ unzip, awscliv2.zip ]
+  - [ ./aws/install ]
+  # Create tests directory with proper ownership and permissions
+  - [ install, -d, -m, '755', -o, ubuntu, -g, ubuntu, /home/ubuntu/tests ]
+  # Clone NooBaa core repository for testing
+  - [ git, clone, https://github.com/noobaa/noobaa-core.git, /home/ubuntu/tests/noobaa-core ]
+  # Install warp runner script to system path with proper permissions
+  - [ install, -m, '755', -o, ubuntu, -g, ubuntu, /home/ubuntu/tests/noobaa-core/tools/ibm_runner_helpers/run_containerized_warp_on_cloud_runner.sh, /usr/local/bin/ ]  
+  # Install Slack notifier script to system path with proper permissions
+  - [ install, -m, '755', -o, ubuntu, -g, ubuntu, /home/ubuntu/tests/noobaa-core/tools/ibm_runner_helpers/slack_notifier.js, /usr/local/bin/ ]
+  # Execute the main warp testing script (this will run the tests and handle results)
+  - [ /usr/local/bin/run_containerized_warp_on_cloud_runner.sh ]

Makefile

@@ -357,15 +357,15 @@ test-warp: tester
 	@$(call create_docker_network)
 	@$(call run_postgres)
 	@echo "\033[1;34mRunning warp tests\033[0m"
-	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/warp/run_warp_on_test_container.sh"
+	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) bash -c "./src/test/external_tests/warp/run_warp_on_test_container.sh $(WARP_ARGS)"
 	@$(call stop_noobaa)
 	@$(call stop_postgres)
 	@$(call remove_docker_network)
 .PHONY: test-warp
 
 test-nc-warp: tester
 	@echo "\033[1;34mRunning warp tests on NC environment\033[0m"
-	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/warp/run_nc_warp_on_test_container.sh"
+	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) bash -c "./src/test/external_tests/warp/run_nc_warp_on_test_container.sh $(WARP_ARGS)"
 .PHONY: test-nc-warp
 
 test-mint: tester

src/test/external_tests/warp/run_nc_warp_on_test_container.sh

@@ -70,6 +70,6 @@ node ./src/test/external_tests/warp/configure_warp.js
 # ====================================================================================
 
 # Run the warp tests
-node ./src/test/external_tests/warp/run_warp.js
+node ./src/test/external_tests/warp/run_warp.js "$@"
 
 # ====================================================================================

src/test/external_tests/warp/run_warp.js

@@ -28,9 +28,11 @@ Usage:
                     set the access key to use for the tests. by default it is set to $access_key.
 --secret-key        <string> 
                     set the secret key to use for the tests. by default it is set to $secret_key.
---obj-size         <string>
+--obj-size          <string>
                     set the object size to use for the tests. by default it is set to 1k.
                     example: 1k, 1m, 1g
+--obj-randsize      <boolean>
+                    set to true to enable random object sizes. by default it is set to false.
 --account-name      <string>
                     set the account name to use for the tests. by default it is set to warp_account.
 `;
@@ -73,6 +75,7 @@ async function run_warp() {
     const duration = argv.duration || '10m';
     const number_of_workers = argv.concurrency || DEFAULT_NUMBER_OF_WORKERS;
     const disable_multipart = argv.disable_multipart || true;
+    const obj_randsize = argv.obj_randsize || false;
     const endpoint = config.ENDPOINT_SSL_PORT;
 
     if (!account_name && !access_key && !secret_key) {
@@ -86,7 +89,7 @@ async function run_warp() {
 
     // TODO - add --benchdata so that the result csv will be saved in logs
     // const warp_logs_dir = WARP_TEST.warp_logs_dir_path;
-    const warp_command = `warp ${op} --host=localhost:${endpoint} --access-key=${access_key} --secret-key=${secret_key} --bucket=${bucket} --obj.size=${obj_size} --duration=${duration} --disable-multipart=${disable_multipart} --tls --insecure --concurrent ${number_of_workers}`;
+    const warp_command = `warp ${op} --host=localhost:${endpoint} --access-key=${access_key} --secret-key=${secret_key} --bucket=${bucket} --obj.size=${obj_size} --duration=${duration} --disable-multipart=${disable_multipart} --tls --insecure --concurrent ${number_of_workers} ${obj_randsize ? ' --obj.randsize' : ''}`;
 
     console.info(`Running warp ${warp_command}`);
     try {

src/test/external_tests/warp/run_warp_on_test_container.sh

@@ -70,6 +70,6 @@ sleep 90
 # ====================================================================================
 
 # Run the warp tests
-node ./src/test/external_tests/warp/run_warp.js
+node ./src/test/external_tests/warp/run_warp.js "$@"
 
 # ====================================================================================

tools/ibm_runner_helpers/run_containerized_warp_on_cloud_runner.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+set -a
+. /etc/warp.env
+set +a
+PS4='+ [run_warp] '
+export HOME=/home/ubuntu
+
+# Global variables for log directory naming
+timestamp=$(date +"%Y-%m-%d-%H-%M-%S")
+log_dir="warp-test-logs-${timestamp}"
+test_logs_path="/home/ubuntu/tests/noobaa-core/logs/warp-test-logs"
+
+send_failure_notification() {
+    local msg="$1"
+    node /usr/local/bin/slack_notifier.js "${SLACK_NIGHTLY_RESULTS_URL}" failure "Warp failed: ${msg}" || true
+}
+
+send_success_notification() {
+    local log_location="$1"
+    node /usr/local/bin/slack_notifier.js "${SLACK_NIGHTLY_RESULTS_URL}" success "Warp success - logs available at: ${log_location}" || true
+}
+
+handle_success() {
+    echo "Uploading logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
+    aws s3 cp "${test_logs_path}" "s3://${WARP_LOGS_BUCKET}/${log_dir}/" --recursive --no-progress --endpoint-url "${IBM_COS_ENDPOINT}"
+    echo "Successfully uploaded logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
+    send_success_notification "s3://${WARP_LOGS_BUCKET}/${log_dir}/"
+}
+
+handle_failure() {
+    echo "Uploading cloud-init logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
+    aws s3 cp "/var/log/cloud-init-output.log" "s3://${WARP_LOGS_BUCKET}/${log_dir}/" --no-progress --endpoint-url "${IBM_COS_ENDPOINT}"
+    echo "Successfully uploaded cloud-init logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
+    send_failure_notification "Run failed, logs available at s3://${WARP_LOGS_BUCKET}/${log_dir}/"
+}
+
+shutdown_vm() {
+    echo "Shutting down VM..."
+    sudo shutdown -P now
+}
+
+trap 'handle_failure "line $LINENO exit $?"' ERR
+trap 'shutdown_vm' EXIT
+
+cd /home/ubuntu/tests/noobaa-core
+
+echo "Building NooBaa images locally..."
+make tester
+
+echo "Creating Warp logs directory..."
+mkdir -p logs/warp-test-logs
+chmod 755 logs/warp-test-logs
+
+echo "Running Warp tests..."
+make test-warp -o tester WARP_ARGS="--duration 3h --obj-size 10m --obj-randsize"
+
+handle_success

tools/ibm_runner_helpers/slack_notifier.js

@@ -0,0 +1,30 @@
+/* Copyright (C) 2025 NooBaa */
+'use strict';
+
+const { IncomingWebhook } = require('@slack/webhook');
+
+async function main() {
+  const [,, webhookUrl, status, message] = process.argv;
+
+  if (!webhookUrl) {
+    process.exit(0);
+  }
+
+  try {
+    const webhook = new IncomingWebhook(webhookUrl);
+    const emoji = status === 'success' ? '✅' : '❌';
+    const text = emoji + ' ' + message;
+
+    // Send message only
+    await webhook.send({
+      text: text
+    });
+
+  } catch (error) {
+    console.error('Notification failed:', error.message);
+  }
+
+  process.exit(0);
+}
+
+main();