lennartalff/backup-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactoring and streamlining

Browse source
This commit is contained in:
Thies Lennart Alff 2025-01-03 13:31:58 +01:00
parent 3c324439c3
commit d3529b6522
Signed by: lennartalff
GPG key ID: 4EC67D34D594104D
4 changed files with 105 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
backup_manager.py Normal file
View file

@ -0,0 +1,63 @@
import gotify
import json
import subprocess
import os
from pathlib import Path
def read_config():
source_path = Path(__file__).resolve()
secrets = source_path.parent / "secrets/paperless.json"
with open(secrets, "r") as f:
config = json.load(f)
return config
class BackupManager:
def __init__(self):
self._config = read_config()
self._remotes = self._config["remotes"]
self._common = self._config["common"]
self._gotify = gotify.Gotify(self._common["GOTIFY_TOKEN"])
def borg_backup(self):
# common config for all remotes
backup_dirs = " ".join(self._common["BACKUP_DIRS"])
exclude_dirs = " ".join(self._common["EXCLUDE_DIRS"])
repo_subdir = self._common["REPO_SUBDIR"]
time_format = self._common["TIME_FORMAT"]
# iterate over all remotes
disabled_remotes = []
for remote in self._remotes:
remote_host = remote["HOSTNAME"]
if not remote["enabled"]:
disabled_remotes.append(remote_host)
continue
local_host = os.uname().nodename
backup_user = remote["BACKUP_USER"]
repo_prefix = remote["REPO_PREFIX"]
borg_env = os.environ.copy()
borg_env["BORG_RSH"] = remote["BORG_RSH"]
borg_env["BORG_PASSPHRASE"] = remote["BORG_PASSPHRASE"]
repo = f"ssh://{backup_user}@{remote_host}/{repo_prefix}/{local_host}/{repo_subdir}::{{{time_format}}}"
cmd = f"borg create -v --stats {repo} {backup_dirs} --exclude {exclude_dirs}"
try:
result = subprocess.run(
cmd,
shell=True,
check=True,
text=True,
capture_output=True,
env=borg_env,
)
except subprocess.CalledProcessError as e:
self._gotify.send_subprocess_error("Backup failed", e)
return False
self._gotify.send_backup_successful(result)
if disabled_remotes:
text = "Skipped disabled remotes:\n" + "\n".join(disabled_remotes)
self._gotify.send_info(title="Skipped remotes", text=text)
return True

43
nextcloud.json.sample
View file

@ -1,16 +1,35 @@
// vim: ft=jsonc
// put this file into the secrets subdir
{
"BORG_RSH": "ssh -i path_to_ssh_key_file",
"BORG_PASSPHRASE": "somepassphrase",
"GOTIFY_TOKEN": "the_gotify_token_for_logging",
"BACKUP_DOCKER_DIR": "dir of the docker compose file",
"BACKUP_DIRS": ["list of dirs to backup"],
"REPO_SUBDIR": "nextcloud",
"BACKUP_USER": "username of the remote repo serving borg",
"EXCLUDE_DIRS": ["list of directories to exclude from the backup"],
"TIME_FORMAT": "utcnow:%Y-%m-%d_%H:%M:%S",
"MYSQL_DB": "database name",
"MYSQL_USER": "database user required for dumping the database",
"MYSQL_PASSWORD": "password required for dumping the database "
"common": {
"GOTIFY_TOKEN": "gotify token",
"BACKUP_DOCKER_DIR": "path to the docker compose file",
"BACKUP_DIRS": ["list of directories to backup"],
"EXCLUDE_DIRS": ["list of directories to exclude"],
"TIME_FORMAT": "utcnow:%Y-%m-%d_%H:%M:%S",
"REPO_SUBDIR": "paperless-ngx",
"MYSQL_DB": "database name",
"MYSQL_USER": "database user required for dumping the database",
"MYSQL_PASSWORD": "password required for dumping the database "
},
"remotes": [
{
"enabled": true,
"HOSTNAME": "myuser.your-storagebox.de:23",
"BORG_RSH": "ssh -i /home/lennartalff/.ssh/borg.ed25519",
"BORG_PASSPHRASE": "the passphrase",
// the resulting repo path is REPO_PREFIX/hostname/REPO_SUBDIR/
"REPO_PREFIX": "backups",
"BACKUP_USER": "u433234"
},
{
"enabled": true,
"HOSTNAME": "mySecondaryBackupServer",
"BORG_RSH": "ssh -i /home/lennartalff/.ssh/borg.ed25519",
"BORG_PASSPHRASE": "the passphrase",
// the resulting repo path is REPO_PREFIX/hostname/REPO_SUBDIR/
"REPO_PREFIX": "backups",
"BACKUP_USER": "u433234"
}
]
}

41
nextcloud_backup
View file

@ -5,6 +5,7 @@ import json
import subprocess
import os
from pathlib import Path
import backup_manager
def read_config():
@ -15,10 +16,9 @@ def read_config():
return config
class BackupManager:
class NextcloudManager(backup_manager.BackupManager):
def __init__(self):
self._config = read_config()
self._gotify = gotify.Gotify(self._config["GOTIFY_TOKEN"])
super().__init__()
def enable_maintenance(self):
cmd = "docker compose exec -i --user 1000:1000 app /var/www/html/occ maintenance:mode --on"
@ -59,9 +59,9 @@ class BackupManager:
return True
def dump_database(self):
password = self._config["MYSQL_PASSWORD"]
user = self._config["MYSQL_USER"]
db = self._config["MYSQL_DB"]
password = self._common["MYSQL_PASSWORD"]
user = self._common["MYSQL_USER"]
db = self._common["MYSQL_DB"]
cmd = f"docker compose exec -i --user 1000:1000 db mariadb-dump --single-transaction --default-character-set=utf8mb4 -h localhost -u {user} --password={password} {db} > db/nextcloud.sql"
try:
@ -78,38 +78,11 @@ class BackupManager:
self._gotify.send_backup_successful(result)
return True
def borg_backup(self):
backup_dirs = " ".join(self._config["BACKUP_DIRS"])
exclude_dirs = " ".join(self._config["EXCLUDE_DIRS"])
repo_subdir = self._config["REPO_SUBDIR"]
time_format = self._config["TIME_FORMAT"]
backup_user = self._config["BACKUP_USER"]
hostname = os.uname().nodename
borg_env = os.environ.copy()
borg_env["BORG_RSH"] = self._config["BORG_RSH"]
borg_env["BORG_PASSPHRASE"] = self._config["BORG_PASSPHRASE"]
repo = f"ssh://{backup_user}@{backup_user}.your-storagebox.de:23/./backups/{hostname}/{repo_subdir}::{{{time_format}}}"
cmd = f"borg create -v --stats {repo} {backup_dirs} --exclude {exclude_dirs}"
try:
result = subprocess.run(
cmd,
shell=True,
check=True,
text=True,
capture_output=True,
env=borg_env,
)
except subprocess.CalledProcessError as e:
self._gotify.send_subprocess_error("Backup failed", e)
return False
self._gotify.send_backup_successful(result)
return True
def main():
config = read_config()
os.chdir(config["BACKUP_DOCKER_DIR"])
backup_manager = BackupManager()
backup_manager = NextcloudManager()
if not backup_manager.enable_maintenance():
backup_manager.disable_maintenance()
exit(1)

53
paperless-ngx_backup
View file

@ -5,6 +5,7 @@ import json
import subprocess
import os
from pathlib import Path
import backup_manager
def read_config():
@ -15,12 +16,9 @@ def read_config():
return config
class BackupManager:
class PaperlessManager(backup_manager.BackupManager):
def __init__(self):
self._config = read_config()
self._remotes = self._config["remotes"]
self._common = self._config["common"]
self._gotify = gotify.Gotify(self._common["GOTIFY_TOKEN"])
super().__init__()
def export_data(self):
cmd = "docker compose exec -it webserver document_exporter ../export -d -f --no-progress-bar"
@ -34,49 +32,6 @@ class BackupManager:
self._gotify.send_success("Data exported.", result)
return True
def borg_backup(self):
# common config for all remotes
backup_dirs = " ".join(self._common["BACKUP_DIRS"])
exclude_dirs = " ".join(self._common["EXCLUDE_DIRS"])
repo_subdir = self._common["REPO_SUBDIR"]
time_format = self._common["TIME_FORMAT"]
disabled_remotes = []
for remote in self._remotes:
remote_host = remote["HOSTNAME"]
if not remote["enabled"]:
disabled_remotes.append(remote_host)
continue
local_host = os.uname().nodename
backup_user = remote["BACKUP_USER"]
repo_prefix = remote["REPO_PREFIX"]
borg_env = os.environ.copy()
borg_env["BORG_RSH"] = remote["BORG_RSH"]
borg_env["BORG_PASSPHRASE"] = remote["BORG_PASSPHRASE"]
repo = f"ssh://{backup_user}@{remote_host}/{repo_prefix}/{local_host}/{repo_subdir}::{{{time_format}}}"
cmd = f"borg create -v --stats {repo} {backup_dirs} --exclude {exclude_dirs}"
try:
result = subprocess.run(
cmd,
shell=True,
check=True,
text=True,
capture_output=True,
env=borg_env,
)
except subprocess.CalledProcessError as e:
self._gotify.send_subprocess_error("Backup failed", e)
return False
self._gotify.send_backup_successful(result)
if disabled_remotes:
text = "Skipped disabled remotes:\n" + "\n".join(disabled_remotes)
self._gotify.send_info(title="Skipped remotes", text=text)
return True
def main():
config = read_config()
@ -84,7 +39,7 @@ def main():
os.chdir(config["common"]["BACKUP_DOCKER_DIR"])
except KeyError:
pass
backup_manager = BackupManager()
backup_manager = PaperlessManager()
if not backup_manager.export_data():
exit(1)
if not backup_manager.borg_backup():