From 86f731ae2ea5d53fd207322f516cc121a73c4c85 Mon Sep 17 00:00:00 2001 From: Julien Noblet Date: Thu, 27 Mar 2025 22:07:44 +0100 Subject: [PATCH 1/4] Add CLEAN and EXTRA_MYSQL/EXTRA_POSTGRESQL options --- README.md | 9 +++++++++ db-auto-backup.py | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 44c24de..1cfcb1d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ Files are backed up uncompressed by default, on the assumption a snapshotting or - `bz2` - `plain` (no compression - the default) +### Clean dumps + +`DROP` statements can be automatically added with `$CLEAN=true`. + +### Extra parameters ! + +Fine manage of your dump policies using extra parameters respectively `$EXTRA_MYSQL` & `$EXTRA_POSTGRESQL`. + ### Example `docker-compose.yml` ```yml @@ -48,6 +56,7 @@ services: environment: - SUCCESS_HOOK_URL=https://hc-ping.com/1234 - INCLUDE_LOGS=true + - EXTRA_MYSQL=--comments --dump-date --lock-all-tables ``` ### Oneshot diff --git a/db-auto-backup.py b/db-auto-backup.py index ca07ff7..5f4a9a8 100755 --- a/db-auto-backup.py +++ b/db-auto-backup.py @@ -97,11 +97,15 @@ def get_success_hook_url() -> Optional[str]: def backup_psql(container: Container) -> str: env = get_container_env(container) user = env.get("POSTGRES_USER", "postgres") - return f"pg_dumpall -U {user}" + extra = EXTRA_POSTGRESQL + if CLEAN: + extra += " --clean --if-exists" + return f"pg_dumpall -U {user} {extra}" def backup_mysql(container: Container) -> str: env = get_container_env(container) + extra = EXTRA_MYSQL # The mariadb container supports both if "MARIADB_ROOT_PASSWORD" in env: @@ -116,7 +120,10 @@ def backup_mysql(container: Container) -> str: else: backup_binary = "mysqldump" - return f"bash -c '{backup_binary} {auth} --all-databases'" + if CLEAN: + extra += " --add-drop-database --add-drop-table --add-drop-trigger" + + return f"bash -c '{backup_binary} {auth} --all-databases {extra}'" def backup_redis(container: Container) -> str: @@ -124,6 +131,7 @@ def backup_redis(container: Container) -> str: Note: `SAVE` command locks the database, which isn't ideal. Hopefully the commit is fast enough! """ + return "sh -c 'redis-cli SAVE > /dev/null && cat /data/dump.rdb'" @@ -159,7 +167,9 @@ def backup_redis(container: Container) -> str: SHOW_PROGRESS = sys.stdout.isatty() COMPRESSION = os.environ.get("COMPRESSION", "plain") INCLUDE_LOGS = bool(os.environ.get("INCLUDE_LOGS")) - +CLEAN = bool(os.environ.get("CLEAN", False)) +EXTRA_PSQL = os.environ.get("EXTRA_POSTGRESQL", "") +EXTRA_MYSQL = os.environ.get("EXTRA_MYSQL", "") def get_backup_provider(container_names: Iterable[str]) -> Optional[BackupProvider]: for name in container_names: From 62cb3f068ff04e1009c553b990523d8fecae05a7 Mon Sep 17 00:00:00 2001 From: Julien Noblet Date: Mon, 31 Mar 2025 21:18:41 +0200 Subject: [PATCH 2/4] Fix CLEAN and EXTRA variables + Linting --- db-auto-backup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/db-auto-backup.py b/db-auto-backup.py index 5f4a9a8..d205e23 100755 --- a/db-auto-backup.py +++ b/db-auto-backup.py @@ -97,15 +97,17 @@ def get_success_hook_url() -> Optional[str]: def backup_psql(container: Container) -> str: env = get_container_env(container) user = env.get("POSTGRES_USER", "postgres") - extra = EXTRA_POSTGRESQL - if CLEAN: + extra = os.environ.get("EXTRA_POSTGRESQL", "") + if bool(os.environ.get("CLEAN")): + print("CLEAN is set, using --clean --if-exists") extra += " --clean --if-exists" + print(f"Using {extra} for backup") return f"pg_dumpall -U {user} {extra}" def backup_mysql(container: Container) -> str: env = get_container_env(container) - extra = EXTRA_MYSQL + extra = os.environ.get("EXTRA_MYSQL", "") # The mariadb container supports both if "MARIADB_ROOT_PASSWORD" in env: @@ -120,7 +122,7 @@ def backup_mysql(container: Container) -> str: else: backup_binary = "mysqldump" - if CLEAN: + if bool(os.environ.get("CLEAN")): extra += " --add-drop-database --add-drop-table --add-drop-trigger" return f"bash -c '{backup_binary} {auth} --all-databases {extra}'" @@ -171,6 +173,7 @@ def backup_redis(container: Container) -> str: EXTRA_PSQL = os.environ.get("EXTRA_POSTGRESQL", "") EXTRA_MYSQL = os.environ.get("EXTRA_MYSQL", "") + def get_backup_provider(container_names: Iterable[str]) -> Optional[BackupProvider]: for name in container_names: for provider in BACKUP_PROVIDERS: From 71da7e86ddf508015b01626a86029a45af7acec1 Mon Sep 17 00:00:00 2001 From: Julien Noblet Date: Tue, 1 Apr 2025 00:04:18 +0200 Subject: [PATCH 3/4] Add e2e for CLEAN option --- tests/tests.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index bf44ac2..9310638 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -128,3 +128,28 @@ def test_get_backup_provider(container_name: str, name: str) -> None: assert provider is not None assert provider.name == name + + +# Have to be run with root privileges to avoid permission issues +def test_backup_runs_clean(run_backup: Callable) -> None: + exit_code, out = run_backup({"CLEAN": "true"}) + assert exit_code == 0, out + assert BACKUP_DIR.is_dir() + assert sorted(normalize_container_name(f.name) for f in BACKUP_DIR.iterdir()) == [ + "docker-db-auto-backup-mariadb-1.sql", + "docker-db-auto-backup-mysql-1.sql", + "docker-db-auto-backup-psql-1.sql", + "docker-db-auto-backup-redis-1.rdb", + ] + + for backup_file in BACKUP_DIR.iterdir(): + if "sql" in backup_file.name: + with open(backup_file, "r") as f: + content = f.read() + if "psql" in backup_file.name: + assert content.find("\nDROP DATABASE") > 0 + assert content.find("\nCREATE DATABASE") > 0 + if "mysql" in backup_file.name or "mariadb" in backup_file.name: + assert content.find("/*!40000 DROP DATABASE IF EXISTS") > 0 + assert content.find("CREATE DATABASE /*!32312 IF NOT EXISTS*/ ") > 0 + assert content.find("CREATE TABLE IF NOT EXISTS ") > 0 From d961ac7ab9d7608810e76d071fbea09046108fa2 Mon Sep 17 00:00:00 2001 From: Julien Noblet Date: Tue, 1 Apr 2025 00:33:30 +0200 Subject: [PATCH 4/4] Fix test_backup_runs_clean to be run without root rights --- tests/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 9310638..1d61be9 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -4,6 +4,7 @@ from typing import Any, Callable from unittest.mock import MagicMock +import docker import pytest BACKUP_DIR = Path.cwd() / "backups" @@ -141,8 +142,12 @@ def test_backup_runs_clean(run_backup: Callable) -> None: "docker-db-auto-backup-psql-1.sql", "docker-db-auto-backup-redis-1.rdb", ] + # Hack for having right on files + docker_client = docker.from_env() + backup_container = docker_client.containers.get("docker-db-auto-backup") for backup_file in BACKUP_DIR.iterdir(): + backup_container.exec_run(["chmod", "o+r", f"/var/backups/{backup_file.name}"]) if "sql" in backup_file.name: with open(backup_file, "r") as f: content = f.read()