From e929c59775a620987bd40caf7a5976ab86204038 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 14:09:30 +0100 Subject: [PATCH 01/18] fix(sftp): add keepalive interval and debug logging to connection config --- src/services/storage/sftp/main.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index d618b7a..f1930ad 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -15,6 +15,7 @@ const { export class SFTPStorage extends StorageClass { private client: SftpClient; + private keepaliveInterval: number = 5_000; // 5 seconds constructor() { super(); @@ -30,6 +31,8 @@ export class SFTPStorage extends StorageClass { host: hostIp, port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, + debug: (msg: string) => logger.debug(msg), + keepaliveInterval: this.keepaliveInterval }; // Authentication: prefer SSH key over password From c799f47e473e45ad178af8f00653e3ad2c99a0e1 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 14:27:38 +0100 Subject: [PATCH 02/18] fix(mysql): default database to ignore --- src/services/backup/mysql/MysqlBackupService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/backup/mysql/MysqlBackupService.ts b/src/services/backup/mysql/MysqlBackupService.ts index 3193136..b99749e 100644 --- a/src/services/backup/mysql/MysqlBackupService.ts +++ b/src/services/backup/mysql/MysqlBackupService.ts @@ -6,7 +6,7 @@ import { statSync } from "node:fs"; const { MYSQL_FOLDER_PATH, MYSQL_IGNORE_DATABASES } = process.env; -const DATABASES_TO_IGNORE = (MYSQL_IGNORE_DATABASES || "").split(','); +const DATABASES_TO_IGNORE = (MYSQL_IGNORE_DATABASES || "information_schema,performance_schema,mysql,sys").split(','); export class MysqlBackupService extends BackupService { SERVICE_NAME = "mysql"; From 8b942e3564cd30bec2a7844a903f26b100053fd0 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 14:53:26 +0100 Subject: [PATCH 03/18] refactor: extract backup service init in backup controller to avoid reconnect --- src/lib/BackupController.ts | 34 ++++++++++++++++++++++++++++++++++ src/main.ts | 18 +----------------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/lib/BackupController.ts b/src/lib/BackupController.ts index d273255..8dd372b 100644 --- a/src/lib/BackupController.ts +++ b/src/lib/BackupController.ts @@ -146,6 +146,36 @@ export class BackupController { return trimmedBackups; } + async initStorageClass() { + try { + await this.backupService.init(); + } catch (error) { + logger.error(`Failed to initialize backup service: ${this.backupService.constructor.name}, error: ${error}`); + await this.alertManager.sendAlert({ + level: AlertLevel.ERROR, + message: `Failed to initialize backup service`, + fields: [ + { name: `Affected service`, value: this.backupService.constructor.name } + ] + }); + } + } + + async closeStorageClass() { + try { + await this.backupService.close(); + } catch (error) { + logger.error(`Failed to close backup service: ${this.backupService.constructor.name}, error: ${error}`); + await this.alertManager.sendAlert({ + level: AlertLevel.ERROR, + message: `Failed to close backup service`, + fields: [ + { name: `Affected service`, value: this.backupService.constructor.name } + ] + }); + } + } + async process() { try { logger.info(`Starting backup process for service: ${this.backupService.SERVICE_NAME}`); @@ -216,10 +246,14 @@ export class BackupController { await EncryptFile(backupFilePath); } + await this.initStorageClass(); + logger.info(`Uploading backup ${backupMetadata.uuid} for ${backupMetadata.parentElement}...`); await this.storageClass.uploadFile(backupFilePath, destinationPath); logger.info(`Uploaded backup file ${backupFilePath} (${backupMetadata.uuid}) to ${destinationPath}`); + await this.closeStorageClass(); + unlinkSync(backupFilePath); } catch (error) { diff --git a/src/main.ts b/src/main.ts index 027bed8..9851543 100644 --- a/src/main.ts +++ b/src/main.ts @@ -21,34 +21,18 @@ const BACKUP_LOCAL_FILES = process.env.BACKUP_LOCAL_FILES === "true"; const ALERT_AFTER_PROCESS = process.env.ALERT_AFTER_PROCESS === "true"; async function processBackup(backupService: BackupService, storageClass: StorageClass, alertManager: AlertManager) { - try { - await backupService.init(); - } catch (error) { - logger.error(`Failed to initialize backup service: ${backupService.constructor.name}, error: ${error}`); - await alertManager.sendAlert({ - level: AlertLevel.ERROR, - message: `Failed to initialize backup service`, - fields: [ - { name: `Affected service`, value: backupService.constructor.name } - ] - }); - throw error; - } - const backupController = new BackupController(backupService, storageClass, alertManager); await backupController.process(); if (ALERT_AFTER_PROCESS) { await alertManager.sendAlert({ level: AlertLevel.INFO, - message: `Backup script has been executed`, + message: `Backup script is being executed`, fields: [ { name: `Affected service`, value: backupService.constructor.name } ] }); } - - await backupService.close(); } async function main() { From 17e8e1a70893d11675a516ccbb572f17a15f75a4 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 15:24:37 +0100 Subject: [PATCH 04/18] Revert "refactor: extract backup service init in backup controller to avoid reconnect" This reverts commit 8b942e3564cd30bec2a7844a903f26b100053fd0. --- src/lib/BackupController.ts | 34 ---------------------------------- src/main.ts | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/src/lib/BackupController.ts b/src/lib/BackupController.ts index 8dd372b..d273255 100644 --- a/src/lib/BackupController.ts +++ b/src/lib/BackupController.ts @@ -146,36 +146,6 @@ export class BackupController { return trimmedBackups; } - async initStorageClass() { - try { - await this.backupService.init(); - } catch (error) { - logger.error(`Failed to initialize backup service: ${this.backupService.constructor.name}, error: ${error}`); - await this.alertManager.sendAlert({ - level: AlertLevel.ERROR, - message: `Failed to initialize backup service`, - fields: [ - { name: `Affected service`, value: this.backupService.constructor.name } - ] - }); - } - } - - async closeStorageClass() { - try { - await this.backupService.close(); - } catch (error) { - logger.error(`Failed to close backup service: ${this.backupService.constructor.name}, error: ${error}`); - await this.alertManager.sendAlert({ - level: AlertLevel.ERROR, - message: `Failed to close backup service`, - fields: [ - { name: `Affected service`, value: this.backupService.constructor.name } - ] - }); - } - } - async process() { try { logger.info(`Starting backup process for service: ${this.backupService.SERVICE_NAME}`); @@ -246,14 +216,10 @@ export class BackupController { await EncryptFile(backupFilePath); } - await this.initStorageClass(); - logger.info(`Uploading backup ${backupMetadata.uuid} for ${backupMetadata.parentElement}...`); await this.storageClass.uploadFile(backupFilePath, destinationPath); logger.info(`Uploaded backup file ${backupFilePath} (${backupMetadata.uuid}) to ${destinationPath}`); - await this.closeStorageClass(); - unlinkSync(backupFilePath); } catch (error) { diff --git a/src/main.ts b/src/main.ts index 9851543..027bed8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -21,18 +21,34 @@ const BACKUP_LOCAL_FILES = process.env.BACKUP_LOCAL_FILES === "true"; const ALERT_AFTER_PROCESS = process.env.ALERT_AFTER_PROCESS === "true"; async function processBackup(backupService: BackupService, storageClass: StorageClass, alertManager: AlertManager) { + try { + await backupService.init(); + } catch (error) { + logger.error(`Failed to initialize backup service: ${backupService.constructor.name}, error: ${error}`); + await alertManager.sendAlert({ + level: AlertLevel.ERROR, + message: `Failed to initialize backup service`, + fields: [ + { name: `Affected service`, value: backupService.constructor.name } + ] + }); + throw error; + } + const backupController = new BackupController(backupService, storageClass, alertManager); await backupController.process(); if (ALERT_AFTER_PROCESS) { await alertManager.sendAlert({ level: AlertLevel.INFO, - message: `Backup script is being executed`, + message: `Backup script has been executed`, fields: [ { name: `Affected service`, value: backupService.constructor.name } ] }); } + + await backupService.close(); } async function main() { From 5a736e63092af4631cf8f820a5cc56a327d910b0 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 15:38:16 +0100 Subject: [PATCH 05/18] fix(sftp): change debug logging level to verbose in connection config --- src/services/storage/sftp/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index f1930ad..7584b8d 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -31,7 +31,7 @@ export class SFTPStorage extends StorageClass { host: hostIp, port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, - debug: (msg: string) => logger.debug(msg), + debug: (msg: string) => logger.verbose(msg), keepaliveInterval: this.keepaliveInterval }; From a60e0b33a63b6a1814d96e71f9e9a70c99191a1d Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 15:52:50 +0100 Subject: [PATCH 06/18] feat(sftp): add keepaliveCountMax parameter --- src/services/storage/sftp/main.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index 7584b8d..33fcbf1 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -16,6 +16,7 @@ const { export class SFTPStorage extends StorageClass { private client: SftpClient; private keepaliveInterval: number = 5_000; // 5 seconds + private keepaliveCountMax: number = 1_000; // Max keepalive attempts constructor() { super(); @@ -32,7 +33,8 @@ export class SFTPStorage extends StorageClass { port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, debug: (msg: string) => logger.verbose(msg), - keepaliveInterval: this.keepaliveInterval + keepaliveInterval: this.keepaliveInterval, + keepaliveCountMax: this.keepaliveCountMax }; // Authentication: prefer SSH key over password From 5d8736ec04cb59c09f91689de1ef6e9703db8ce9 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Wed, 21 Jan 2026 16:14:22 +0100 Subject: [PATCH 07/18] feat(sftp): implements always connect / disconnect before all methods --- src/services/storage/sftp/main.ts | 35 +++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index 33fcbf1..fb4954e 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -60,57 +60,84 @@ export class SFTPStorage extends StorageClass { logger.info(`Connected to SFTP server: ${hostIp}:${config.port}`); } + async disconnect() { + await this.client.end(); + logger.info(`Disconnected from SFTP server`); + } + async deleteFile(filePath: string) { + await this.connect(); + try { await this.client.delete(filePath); logger.debug(`Deleted file: ${filePath}`); } catch (error) { logger.error(`Failed to delete file ${filePath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async uploadFile(filePath: string, destination: string) { + await this.connect(); + try { await this.client.put(filePath, destination); logger.debug(`Uploaded file: ${filePath}, to: ${destination}`); } catch (error) { logger.error(`Failed to upload file ${filePath} to ${destination}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async createFolder(folderPath: string) { + await this.connect(); + try { await this.client.mkdir(folderPath, true); // recursive = true logger.debug(`Created folder: ${folderPath}`); } catch (error) { logger.error(`Failed to create folder ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async deleteFolder(folderPath: string) { + await this.connect(); + try { await this.client.rmdir(folderPath, true); // recursive = true logger.debug(`Deleted folder: ${folderPath}`); } catch (error) { logger.error(`Failed to delete folder ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async folderExists(folderPath: string): Promise { + await this.connect(); + try { const stat = await this.client.stat(folderPath); return stat.isDirectory; } catch (error) { // If stat fails, the folder doesn't exist return false; + } finally { + await this.disconnect(); } } async folderSizeBytes(folderPath: string) { + await this.connect(); + try { const list = await this.client.list(folderPath); let totalSize = 0; @@ -129,10 +156,14 @@ export class SFTPStorage extends StorageClass { } catch (error) { logger.error(`Failed to calculate folder size for ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async listFiles(folderPath: string) { + await this.connect(); + try { const list = await this.client.list(folderPath); @@ -146,14 +177,14 @@ export class SFTPStorage extends StorageClass { } catch (error) { logger.error(`Failed to list files in folder ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async close() { - await this.client.end(); } async init() { - await this.connect(); } } From e96bf4b378e0f709caa62741cbf41a8ea9d256bb Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Wed, 21 Jan 2026 22:15:15 +0100 Subject: [PATCH 08/18] fix(sftp): try to improve big files upload --- src/services/storage/sftp/main.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index fb4954e..b869f41 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -32,7 +32,7 @@ export class SFTPStorage extends StorageClass { host: hostIp, port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, - debug: (msg: string) => logger.verbose(msg), + debug: (msg: string) => logger.silly(msg), keepaliveInterval: this.keepaliveInterval, keepaliveCountMax: this.keepaliveCountMax }; @@ -83,7 +83,13 @@ export class SFTPStorage extends StorageClass { await this.connect(); try { - await this.client.put(filePath, destination); + const readStream = fs.createReadStream(filePath); + await this.client.put(readStream, destination, { + concurrency: 64, + step: (transferred, chunk, total) => { + logger.verbose(`Uploading ${filePath}: ${((transferred / total) * 100).toFixed(2)}%`); + } + }); logger.debug(`Uploaded file: ${filePath}, to: ${destination}`); } catch (error) { logger.error(`Failed to upload file ${filePath} to ${destination}: ${error}`); From d9387b10e1f174967f10664b2e7c2a54359c2868 Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Wed, 21 Jan 2026 22:19:46 +0100 Subject: [PATCH 09/18] fix(sftp): build issue --- src/services/storage/sftp/main.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index b869f41..9929c9a 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -85,7 +85,6 @@ export class SFTPStorage extends StorageClass { try { const readStream = fs.createReadStream(filePath); await this.client.put(readStream, destination, { - concurrency: 64, step: (transferred, chunk, total) => { logger.verbose(`Uploading ${filePath}: ${((transferred / total) * 100).toFixed(2)}%`); } From be67fb4efc223e1b8639dbbcbf2b5b0d8ea6a9d2 Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Wed, 21 Jan 2026 22:21:15 +0100 Subject: [PATCH 10/18] fix(sftp): build issue - again --- src/services/storage/sftp/main.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index 9929c9a..c952231 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -32,7 +32,7 @@ export class SFTPStorage extends StorageClass { host: hostIp, port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, - debug: (msg: string) => logger.silly(msg), + debug: (msg: string) => logger.debug(msg), keepaliveInterval: this.keepaliveInterval, keepaliveCountMax: this.keepaliveCountMax }; @@ -84,11 +84,7 @@ export class SFTPStorage extends StorageClass { try { const readStream = fs.createReadStream(filePath); - await this.client.put(readStream, destination, { - step: (transferred, chunk, total) => { - logger.verbose(`Uploading ${filePath}: ${((transferred / total) * 100).toFixed(2)}%`); - } - }); + await this.client.put(readStream, destination); logger.debug(`Uploaded file: ${filePath}, to: ${destination}`); } catch (error) { logger.error(`Failed to upload file ${filePath} to ${destination}: ${error}`); From 05a12224332b94ee65f8952f3143c874e38b751e Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Wed, 21 Jan 2026 22:54:44 +0100 Subject: [PATCH 11/18] fix(sftp): try to use fastPut --- src/services/storage/sftp/main.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index c952231..01c6934 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -32,7 +32,7 @@ export class SFTPStorage extends StorageClass { host: hostIp, port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, - debug: (msg: string) => logger.debug(msg), + debug: (msg: string) => logger.silly(msg), keepaliveInterval: this.keepaliveInterval, keepaliveCountMax: this.keepaliveCountMax }; @@ -83,8 +83,13 @@ export class SFTPStorage extends StorageClass { await this.connect(); try { - const readStream = fs.createReadStream(filePath); - await this.client.put(readStream, destination); + await this.client.fastPut(filePath, destination, { + concurrency: 64, + chunkSize: 32768, + step: (totalTransferred, chunk, total) => { + logger.debug(`Upload progress: ${((totalTransferred / total) * 100).toFixed(2)}%`); + } + }) logger.debug(`Uploaded file: ${filePath}, to: ${destination}`); } catch (error) { logger.error(`Failed to upload file ${filePath} to ${destination}: ${error}`); From ee49ff3e3579cf94a54a0f00c26bc3944906c515 Mon Sep 17 00:00:00 2001 From: "Hugo H." Date: Fri, 23 Jan 2026 14:14:22 +0100 Subject: [PATCH 12/18] fix(encryption): Native Node streams are no longer supported error --- src/services/files/encryption.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/services/files/encryption.ts b/src/services/files/encryption.ts index 2304fec..00fab48 100644 --- a/src/services/files/encryption.ts +++ b/src/services/files/encryption.ts @@ -2,6 +2,7 @@ import "dotenv/config"; import { createMessage, encrypt, readKey } from "openpgp"; import { readFile, writeFile, rename } from "node:fs/promises"; import { createReadStream, createWriteStream } from "node:fs"; +import { Readable, Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { logger } from "../log"; @@ -21,20 +22,25 @@ export async function EncryptFile(filePath: string): Promise { logger.debug(`Encrypting file ${filePath}...`); // Use streaming to handle large files without loading them entirely into memory - const readStream = createReadStream(filePath); + const nodeReadStream = createReadStream(filePath); + // Convert Node.js stream to WebStream (required by openpgp v6+) + const webReadStream = Readable.toWeb(nodeReadStream) as ReadableStream; const tempFilePath = `${filePath}.tmp`; const writeStream = createWriteStream(tempFilePath); const encryptedStream = await encrypt({ - message: await createMessage({ binary: readStream }), + message: await createMessage({ binary: webReadStream }), encryptionKeys: publicKey, format: 'armored' }); logger.debug(`File ${filePath} encrypted successfully, writing to temporary file...`); + // Convert WebStream back to Node.js stream for pipeline + const nodeEncryptedStream = Readable.fromWeb(encryptedStream as import("stream/web").ReadableStream); + // Pipe the encrypted stream to the output file - await pipeline(encryptedStream, writeStream); + await pipeline(nodeEncryptedStream, writeStream); // Replace the original file with the encrypted one await rename(tempFilePath, filePath); From 4503f7d6db87b3c09a710cc8c110ef33d72349d0 Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Mon, 16 Feb 2026 23:07:46 +0100 Subject: [PATCH 13/18] fix(rsync): now ignore compression for already compressed files --- src/services/backup/rsync/utils.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/services/backup/rsync/utils.ts b/src/services/backup/rsync/utils.ts index 1b0f4e4..c0e3928 100644 --- a/src/services/backup/rsync/utils.ts +++ b/src/services/backup/rsync/utils.ts @@ -151,6 +151,11 @@ async function runCommand(command: string, args: string[], options?: CommandOpti }); } +export function isArchiveFile(fileName: string) { + const extensions = [".tar.gz", ".tgz", ".tar"]; + return extensions.some((ext) => fileName.endsWith(ext)); +} + function buildRsyncArgs(target: RsyncTarget, destination: string) { const args = ["-az"]; @@ -188,8 +193,8 @@ export async function createArchiveForTarget(target: RsyncTarget) { const timestamp = buildTimestamp(date); const workingDirectory = join(RSYNC_TMP_ROOT, `${sanitizedName}-${timestamp}`); - const archiveName = `${sanitizedName}-${timestamp}.tar.gz`; - const archivePath = join(RSYNC_TMP_ROOT, archiveName); + let archiveName = `${sanitizedName}-${timestamp}.tar.gz`; + let archivePath = join(RSYNC_TMP_ROOT, archiveName); mkdirSync(workingDirectory, { recursive: true }); @@ -198,6 +203,20 @@ export async function createArchiveForTarget(target: RsyncTarget) { const rsyncArgs = buildRsyncArgs(target, workingDirectory); await runCommand("rsync", rsyncArgs, { logPrefix: "rsync" }); + if (!isArchiveFile(archiveName)) { + archiveName += ".tar.gz"; + archivePath = join(RSYNC_TMP_ROOT, archiveName); + + return { + archiveName, + archivePath, + sanitizedName: archiveName, + timestamp, + size: statSync(archivePath).size, + date + }; + } + logger.info(`[rsync] Creating archive for "${target.name}".`); await runCommand("tar", ["-czf", archivePath, "-C", workingDirectory, "."], { logPrefix: "tar" }); From 365162670b13177510853e318ebd6c356b5543d6 Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Mon, 16 Feb 2026 23:40:10 +0100 Subject: [PATCH 14/18] feat(rsync): add support for single and multi-backup modes --- README.md | 37 ++++++ .../backup/rsync/RsyncBackupService.ts | 120 ++++++++++++++++-- src/services/backup/rsync/utils.ts | 73 +++++++++-- 3 files changed, 208 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5717a1a..edf2e0b 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,43 @@ The `PERIODIC_BACKUP_RETENTION` setting allows you to define custom retention po | `RSYNC_SSH_OPTIONS` | Additional SSH options appended to the rsync SSH command. Useful for disabling host key checks during testing. | _(empty)_ | | `RSYNC_EXTRA_ARGS` | Additional rsync arguments (space separated, double quotes supported for paths or arguments that contain spaces) | _(empty)_ | +#### Rsync Backup Modes + +The rsync backup service supports two different backup modes depending on the path configuration: + +**Single Backup Mode (without `/*`):** + +When `RSYNC_TARGET_PATH` does not end with `/*`, the entire directory content is backed up as a single archive with a timestamp. A new backup is created at each execution. + +```bash +RSYNC_TARGET_PATH=/var/www/mysite +``` + +Result: +- Creates: `mysite-2026-02-16-14-30-00.tar.gz` +- Behavior: New backup at each execution +- Storage: `rsync/mysite/mysite-2026-02-16-14-30-00.tar.gz` + +**Multi-Backup Mode (with `/*`):** + +When `RSYNC_TARGET_PATH` ends with `/*`, each file and subdirectory is backed up individually **without timestamp**. This prevents duplicate backups: if a backup already exists for an item, it is skipped. + +```bash +RSYNC_TARGET_PATH=/var/www/* +``` + +Result (assuming `/var/www/` contains `site1`, `site2`, `site3`): +- Creates: `site1.tar.gz`, `site2.tar.gz`, `site3.tar.gz` +- Behavior: Backup only created if file doesn't exist +- Storage: `rsync/target-name/site1/site1.tar.gz`, `rsync/target-name/site2/site2.tar.gz`, etc. + +**Use Cases:** + +- **Single mode**: For backing up a complete application with versioning (database backups, complete application snapshots) +- **Multi mode**: For backing up multiple independent sites/projects where you only want to backup once and skip duplicates (web hosting directories, user folders) + +#### Development Setup + The default development `.env` values point `RSYNC_TARGET_HOST` to this container (`rsync`) and mount the generated SSH private key at `/app/docker_keys/id_ed25519`. The files served over rsync live in `docker/rsync/data`. To generate the development key pair, run the helper script and recreate the container: diff --git a/src/services/backup/rsync/RsyncBackupService.ts b/src/services/backup/rsync/RsyncBackupService.ts index 29782db..e277af2 100644 --- a/src/services/backup/rsync/RsyncBackupService.ts +++ b/src/services/backup/rsync/RsyncBackupService.ts @@ -1,15 +1,22 @@ import { BackupService } from "../BackupService"; import { BackupFileMetadata } from "../types/BackupFileMetadata"; -import { loadRsyncTarget, createArchiveForTarget, RsyncTarget } from "./utils"; +import { loadRsyncTarget, createArchiveForTarget, RsyncTarget, listRemoteFiles, sanitizeName, buildTimestamp } from "./utils"; import { logger } from "../../log"; const { RSYNC_FOLDER_PATH, BACKUP_RSYNC } = process.env; +type PendingBackup = { + target: RsyncTarget; + itemName: string; + specificPath: string; +}; + export class RsyncBackupService extends BackupService { SERVICE_NAME = "rsync"; FOLDER_PATH = RSYNC_FOLDER_PATH || "rsync"; private target: RsyncTarget | undefined; + private pendingBackups: Map = new Map(); async init() { this.target = loadRsyncTarget(); @@ -24,22 +31,107 @@ export class RsyncBackupService extends BackupService { } try { - const archive = await createArchiveForTarget(this.target); - - return [ - { - parentElement: this.target.name, - destinationFolder: `${this.FOLDER_PATH}/${archive.sanitizedName}`, - fileName: archive.archiveName, - uuid: `rsync-${archive.sanitizedName}-${archive.timestamp}`, - size: archive.size, - date: archive.date, - localPath: archive.archivePath - } - ]; + // Check if path ends with /* + if (this.target.path.endsWith("/*")) { + return await this.getMultipleBackups(); + } else { + return await this.getSingleBackup(); + } } catch (error) { logger.error(`[rsync] Failed to create archive: ${error}`); throw error; } } + + private async getSingleBackup(): Promise { + if (!this.target) { + return []; + } + + const archive = await createArchiveForTarget(this.target); + + return [ + { + parentElement: this.target.name, + destinationFolder: `${this.FOLDER_PATH}/${archive.sanitizedName}`, + fileName: archive.archiveName, + uuid: `rsync-${archive.sanitizedName}-${archive.timestamp}`, + size: archive.size, + date: archive.date, + localPath: archive.archivePath + } + ]; + } + + private async getMultipleBackups(): Promise { + if (!this.target) { + return []; + } + + // Remove /* from the path to get the base directory + const basePath = this.target.path.slice(0, -2); + + logger.info(`[rsync] Listing files in remote directory: ${basePath}`); + const remoteFiles = await listRemoteFiles(this.target, basePath); + + if (remoteFiles.length === 0) { + logger.warn(`[rsync] No files found in remote directory: ${basePath}`); + return []; + } + + logger.info(`[rsync] Found ${remoteFiles.length} items to backup individually`); + + const result: BackupFileMetadata[] = []; + const date = new Date(); + const timestamp = buildTimestamp(date); + + for (const itemName of remoteFiles) { + const sanitizedItemName = sanitizeName(itemName); + const specificPath = `${basePath}/${itemName}`; + // Use timestamp only in UUID for internal tracking, not in filename + const uuid = `rsync-${sanitizeName(this.target.name)}-${sanitizedItemName}-${timestamp}`; + + // Store the pending backup info for later download + this.pendingBackups.set(uuid, { + target: this.target, + itemName, + specificPath + }); + + result.push({ + parentElement: `${this.target.name} - ${itemName}`, + destinationFolder: `${this.FOLDER_PATH}/${sanitizeName(this.target.name)}/${sanitizedItemName}`, + // No timestamp in filename: this allows BackupController to detect existing backups + fileName: `${sanitizedItemName}.tar.gz`, + uuid, + size: 0, // Size will be known after archive creation + date + }); + } + + return result; + } + + async downloadBackup(backupMetadata: BackupFileMetadata): Promise { + // Check if this is a pending backup (multi-file mode) + const pendingBackup = this.pendingBackups.get(backupMetadata.uuid); + + if (pendingBackup) { + // Create the archive now + logger.info(`[rsync] Creating archive for item: ${pendingBackup.itemName}`); + const archive = await createArchiveForTarget( + pendingBackup.target, + pendingBackup.specificPath, + pendingBackup.itemName + ); + + // Remove from pending backups + this.pendingBackups.delete(backupMetadata.uuid); + + return archive.archivePath; + } + + // Default behavior for single backup mode + return backupMetadata.localPath; + } } \ No newline at end of file diff --git a/src/services/backup/rsync/utils.ts b/src/services/backup/rsync/utils.ts index c0e3928..8ba52f8 100644 --- a/src/services/backup/rsync/utils.ts +++ b/src/services/backup/rsync/utils.ts @@ -156,9 +156,7 @@ export function isArchiveFile(fileName: string) { return extensions.some((ext) => fileName.endsWith(ext)); } -function buildRsyncArgs(target: RsyncTarget, destination: string) { - const args = ["-az"]; - +function buildSshCommand(target: RsyncTarget): string[] { const sshParts = ["ssh"]; if (target.port) { @@ -173,6 +171,14 @@ function buildRsyncArgs(target: RsyncTarget, destination: string) { sshParts.push(option); } + return sshParts; +} + +function buildRsyncArgs(target: RsyncTarget, destination: string, specificPath?: string) { + const args = ["-az"]; + + const sshParts = buildSshCommand(target); + for (const exclude of target.excludes) { args.push("--exclude", exclude); } @@ -181,14 +187,62 @@ function buildRsyncArgs(target: RsyncTarget, destination: string) { args.push("-e", sshParts.join(" ")); - const remoteSpec = `${target.user ? `${target.user}@` : ""}${formatHost(target.host)}:${target.path}`; + const remotePath = specificPath || target.path; + const remoteSpec = `${target.user ? `${target.user}@` : ""}${formatHost(target.host)}:${remotePath}`; args.push(remoteSpec, destination); return args; } -export async function createArchiveForTarget(target: RsyncTarget) { - const sanitizedName = sanitizeName(target.name); +export async function listRemoteFiles(target: RsyncTarget, remotePath: string): Promise { + const sshCommand = buildSshCommand(target); + const host = formatHost(target.host); + const remoteHost = target.user ? `${target.user}@${host}` : host; + + // Use find to list only direct children (maxdepth 1) and get basename + const findCommand = `find "${remotePath}" -mindepth 1 -maxdepth 1 -printf '%f\\n'`; + + const args = [...sshCommand, remoteHost, findCommand]; + + return new Promise((resolve, reject) => { + const child = spawn("ssh", args, { + stdio: ["ignore", "pipe", "pipe"] + }); + + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", (data) => { + stdout += data.toString(); + }); + + child.stderr?.on("data", (data) => { + stderr += data.toString(); + }); + + child.on("error", (error) => { + logger.error(`[rsync] Failed to list remote files: ${error}`); + reject(error); + }); + + child.on("close", (code) => { + if (code === 0) { + const files = stdout + .trim() + .split("\n") + .filter(line => line.length > 0); + resolve(files); + return; + } + + logger.error(`[rsync] Failed to list remote files. Exit code ${code}. Stderr: ${stderr.trim()}`); + reject(new Error(`Failed to list remote files: ${stderr.trim()}`)); + }); + }); +} + +export async function createArchiveForTarget(target: RsyncTarget, specificPath?: string, itemName?: string) { + const sanitizedName = itemName ? sanitizeName(itemName) : sanitizeName(target.name); const date = new Date(); const timestamp = buildTimestamp(date); @@ -198,9 +252,12 @@ export async function createArchiveForTarget(target: RsyncTarget) { mkdirSync(workingDirectory, { recursive: true }); + const remotePath = specificPath || target.path; + const displayPath = specificPath ? `${target.path}/${specificPath}` : target.path; + try { - logger.info(`[rsync] Starting sync for "${target.name}" (${target.host}:${target.path}).`); - const rsyncArgs = buildRsyncArgs(target, workingDirectory); + logger.info(`[rsync] Starting sync for "${target.name}" (${target.host}:${displayPath}).`); + const rsyncArgs = buildRsyncArgs(target, workingDirectory, remotePath); await runCommand("rsync", rsyncArgs, { logPrefix: "rsync" }); if (!isArchiveFile(archiveName)) { From 13d9e691bbb70ffefd1b476560e6976bff9e46cc Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Mon, 16 Feb 2026 23:42:53 +0100 Subject: [PATCH 15/18] fix(rsync): update path check to handle wildcard correctly --- src/services/backup/rsync/RsyncBackupService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/backup/rsync/RsyncBackupService.ts b/src/services/backup/rsync/RsyncBackupService.ts index e277af2..0d5aaa1 100644 --- a/src/services/backup/rsync/RsyncBackupService.ts +++ b/src/services/backup/rsync/RsyncBackupService.ts @@ -31,8 +31,8 @@ export class RsyncBackupService extends BackupService { } try { - // Check if path ends with /* - if (this.target.path.endsWith("/*")) { + // Check if path ends with * + if (this.target.path.endsWith("*")) { return await this.getMultipleBackups(); } else { return await this.getSingleBackup(); @@ -69,7 +69,7 @@ export class RsyncBackupService extends BackupService { } // Remove /* from the path to get the base directory - const basePath = this.target.path.slice(0, -2); + const basePath = this.target.path.endsWith("/*") ? this.target.path.slice(0, -2) : this.target.path.endsWith("*") ? this.target.path.slice(0, -1) : this.target.path; logger.info(`[rsync] Listing files in remote directory: ${basePath}`); const remoteFiles = await listRemoteFiles(this.target, basePath); From c399f9cce6a7b261a4da6255fa2d30c0f55bc182 Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Tue, 17 Feb 2026 00:01:03 +0100 Subject: [PATCH 16/18] fix(rsync): fix some errors for listRemoteFiles --- src/services/backup/rsync/utils.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/services/backup/rsync/utils.ts b/src/services/backup/rsync/utils.ts index 8ba52f8..209ede3 100644 --- a/src/services/backup/rsync/utils.ts +++ b/src/services/backup/rsync/utils.ts @@ -199,13 +199,13 @@ export async function listRemoteFiles(target: RsyncTarget, remotePath: string): const host = formatHost(target.host); const remoteHost = target.user ? `${target.user}@${host}` : host; - // Use find to list only direct children (maxdepth 1) and get basename const findCommand = `find "${remotePath}" -mindepth 1 -maxdepth 1 -printf '%f\\n'`; - const args = [...sshCommand, remoteHost, findCommand]; + const [bin, ...sshArgs] = sshCommand; + const args = [...sshArgs, remoteHost, findCommand]; return new Promise((resolve, reject) => { - const child = spawn("ssh", args, { + const child = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] }); @@ -231,12 +231,15 @@ export async function listRemoteFiles(target: RsyncTarget, remotePath: string): .trim() .split("\n") .filter(line => line.length > 0); + + logger.debug(`[rsync] Remote files found: ${files.length}`); resolve(files); return; } - logger.error(`[rsync] Failed to list remote files. Exit code ${code}. Stderr: ${stderr.trim()}`); - reject(new Error(`Failed to list remote files: ${stderr.trim()}`)); + const errorMsg = stderr.trim() || "Unknown SSH error"; + logger.error(`[rsync] Failed to list remote files. Exit code ${code}. Stderr: ${errorMsg}`); + reject(new Error(`Failed to list remote files: ${errorMsg}`)); }); }); } From 52031699f99ed44a15b6172681992a63865dec6f Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Tue, 17 Feb 2026 00:04:48 +0100 Subject: [PATCH 17/18] fix(rsync): simplify error handling and streamline data processing in listRemoteFiles --- src/services/backup/rsync/utils.ts | 32 ++++++------------------------ 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/src/services/backup/rsync/utils.ts b/src/services/backup/rsync/utils.ts index 209ede3..c11b5e3 100644 --- a/src/services/backup/rsync/utils.ts +++ b/src/services/backup/rsync/utils.ts @@ -205,41 +205,21 @@ export async function listRemoteFiles(target: RsyncTarget, remotePath: string): const args = [...sshArgs, remoteHost, findCommand]; return new Promise((resolve, reject) => { - const child = spawn(bin, args, { - stdio: ["ignore", "pipe", "pipe"] - }); + const child = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; - child.stdout?.on("data", (data) => { - stdout += data.toString(); - }); - - child.stderr?.on("data", (data) => { - stderr += data.toString(); - }); - - child.on("error", (error) => { - logger.error(`[rsync] Failed to list remote files: ${error}`); - reject(error); - }); + child.stdout?.on("data", (data) => stdout += data.toString()); + child.stderr?.on("data", (data) => stderr += data.toString()); child.on("close", (code) => { if (code === 0) { - const files = stdout - .trim() - .split("\n") - .filter(line => line.length > 0); - - logger.debug(`[rsync] Remote files found: ${files.length}`); + const files = stdout.trim().split("\n").filter(l => l.length > 0); resolve(files); - return; + } else { + reject(new Error(`SSH Exit ${code}: ${stderr.trim()}`)); } - - const errorMsg = stderr.trim() || "Unknown SSH error"; - logger.error(`[rsync] Failed to list remote files. Exit code ${code}. Stderr: ${errorMsg}`); - reject(new Error(`Failed to list remote files: ${errorMsg}`)); }); }); } From 6d80f20ea09fca7d00371b9e5475c842bf40b287 Mon Sep 17 00:00:00 2001 From: "Hugo H." <51268820+hugoheml@users.noreply.github.com> Date: Tue, 17 Feb 2026 00:08:39 +0100 Subject: [PATCH 18/18] fix(rsync): improve error reporting in listRemoteFiles by including stdout in rejection --- src/services/backup/rsync/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/backup/rsync/utils.ts b/src/services/backup/rsync/utils.ts index c11b5e3..3302324 100644 --- a/src/services/backup/rsync/utils.ts +++ b/src/services/backup/rsync/utils.ts @@ -218,7 +218,7 @@ export async function listRemoteFiles(target: RsyncTarget, remotePath: string): const files = stdout.trim().split("\n").filter(l => l.length > 0); resolve(files); } else { - reject(new Error(`SSH Exit ${code}: ${stderr.trim()}`)); + reject(new Error(`SSH Exit ${code}: ${stdout} - ${stderr.trim()}`)); } }); });