diff --git a/README.md b/README.md index 5717a1a..edf2e0b 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,43 @@ The `PERIODIC_BACKUP_RETENTION` setting allows you to define custom retention po | `RSYNC_SSH_OPTIONS` | Additional SSH options appended to the rsync SSH command. Useful for disabling host key checks during testing. | _(empty)_ | | `RSYNC_EXTRA_ARGS` | Additional rsync arguments (space separated, double quotes supported for paths or arguments that contain spaces) | _(empty)_ | +#### Rsync Backup Modes + +The rsync backup service supports two different backup modes depending on the path configuration: + +**Single Backup Mode (without `/*`):** + +When `RSYNC_TARGET_PATH` does not end with `/*`, the entire directory content is backed up as a single archive with a timestamp. A new backup is created at each execution. + +```bash +RSYNC_TARGET_PATH=/var/www/mysite +``` + +Result: +- Creates: `mysite-2026-02-16-14-30-00.tar.gz` +- Behavior: New backup at each execution +- Storage: `rsync/mysite/mysite-2026-02-16-14-30-00.tar.gz` + +**Multi-Backup Mode (with `/*`):** + +When `RSYNC_TARGET_PATH` ends with `/*`, each file and subdirectory is backed up individually **without timestamp**. This prevents duplicate backups: if a backup already exists for an item, it is skipped. + +```bash +RSYNC_TARGET_PATH=/var/www/* +``` + +Result (assuming `/var/www/` contains `site1`, `site2`, `site3`): +- Creates: `site1.tar.gz`, `site2.tar.gz`, `site3.tar.gz` +- Behavior: Backup only created if file doesn't exist +- Storage: `rsync/target-name/site1/site1.tar.gz`, `rsync/target-name/site2/site2.tar.gz`, etc. + +**Use Cases:** + +- **Single mode**: For backing up a complete application with versioning (database backups, complete application snapshots) +- **Multi mode**: For backing up multiple independent sites/projects where you only want to backup once and skip duplicates (web hosting directories, user folders) + +#### Development Setup + The default development `.env` values point `RSYNC_TARGET_HOST` to this container (`rsync`) and mount the generated SSH private key at `/app/docker_keys/id_ed25519`. The files served over rsync live in `docker/rsync/data`. To generate the development key pair, run the helper script and recreate the container: diff --git a/src/services/backup/mysql/MysqlBackupService.ts b/src/services/backup/mysql/MysqlBackupService.ts index 3193136..b99749e 100644 --- a/src/services/backup/mysql/MysqlBackupService.ts +++ b/src/services/backup/mysql/MysqlBackupService.ts @@ -6,7 +6,7 @@ import { statSync } from "node:fs"; const { MYSQL_FOLDER_PATH, MYSQL_IGNORE_DATABASES } = process.env; -const DATABASES_TO_IGNORE = (MYSQL_IGNORE_DATABASES || "").split(','); +const DATABASES_TO_IGNORE = (MYSQL_IGNORE_DATABASES || "information_schema,performance_schema,mysql,sys").split(','); export class MysqlBackupService extends BackupService { SERVICE_NAME = "mysql"; diff --git a/src/services/backup/rsync/RsyncBackupService.ts b/src/services/backup/rsync/RsyncBackupService.ts index 29782db..0d5aaa1 100644 --- a/src/services/backup/rsync/RsyncBackupService.ts +++ b/src/services/backup/rsync/RsyncBackupService.ts @@ -1,15 +1,22 @@ import { BackupService } from "../BackupService"; import { BackupFileMetadata } from "../types/BackupFileMetadata"; -import { loadRsyncTarget, createArchiveForTarget, RsyncTarget } from "./utils"; +import { loadRsyncTarget, createArchiveForTarget, RsyncTarget, listRemoteFiles, sanitizeName, buildTimestamp } from "./utils"; import { logger } from "../../log"; const { RSYNC_FOLDER_PATH, BACKUP_RSYNC } = process.env; +type PendingBackup = { + target: RsyncTarget; + itemName: string; + specificPath: string; +}; + export class RsyncBackupService extends BackupService { SERVICE_NAME = "rsync"; FOLDER_PATH = RSYNC_FOLDER_PATH || "rsync"; private target: RsyncTarget | undefined; + private pendingBackups: Map = new Map(); async init() { this.target = loadRsyncTarget(); @@ -24,22 +31,107 @@ export class RsyncBackupService extends BackupService { } try { - const archive = await createArchiveForTarget(this.target); - - return [ - { - parentElement: this.target.name, - destinationFolder: `${this.FOLDER_PATH}/${archive.sanitizedName}`, - fileName: archive.archiveName, - uuid: `rsync-${archive.sanitizedName}-${archive.timestamp}`, - size: archive.size, - date: archive.date, - localPath: archive.archivePath - } - ]; + // Check if path ends with * + if (this.target.path.endsWith("*")) { + return await this.getMultipleBackups(); + } else { + return await this.getSingleBackup(); + } } catch (error) { logger.error(`[rsync] Failed to create archive: ${error}`); throw error; } } + + private async getSingleBackup(): Promise { + if (!this.target) { + return []; + } + + const archive = await createArchiveForTarget(this.target); + + return [ + { + parentElement: this.target.name, + destinationFolder: `${this.FOLDER_PATH}/${archive.sanitizedName}`, + fileName: archive.archiveName, + uuid: `rsync-${archive.sanitizedName}-${archive.timestamp}`, + size: archive.size, + date: archive.date, + localPath: archive.archivePath + } + ]; + } + + private async getMultipleBackups(): Promise { + if (!this.target) { + return []; + } + + // Remove /* from the path to get the base directory + const basePath = this.target.path.endsWith("/*") ? this.target.path.slice(0, -2) : this.target.path.endsWith("*") ? this.target.path.slice(0, -1) : this.target.path; + + logger.info(`[rsync] Listing files in remote directory: ${basePath}`); + const remoteFiles = await listRemoteFiles(this.target, basePath); + + if (remoteFiles.length === 0) { + logger.warn(`[rsync] No files found in remote directory: ${basePath}`); + return []; + } + + logger.info(`[rsync] Found ${remoteFiles.length} items to backup individually`); + + const result: BackupFileMetadata[] = []; + const date = new Date(); + const timestamp = buildTimestamp(date); + + for (const itemName of remoteFiles) { + const sanitizedItemName = sanitizeName(itemName); + const specificPath = `${basePath}/${itemName}`; + // Use timestamp only in UUID for internal tracking, not in filename + const uuid = `rsync-${sanitizeName(this.target.name)}-${sanitizedItemName}-${timestamp}`; + + // Store the pending backup info for later download + this.pendingBackups.set(uuid, { + target: this.target, + itemName, + specificPath + }); + + result.push({ + parentElement: `${this.target.name} - ${itemName}`, + destinationFolder: `${this.FOLDER_PATH}/${sanitizeName(this.target.name)}/${sanitizedItemName}`, + // No timestamp in filename: this allows BackupController to detect existing backups + fileName: `${sanitizedItemName}.tar.gz`, + uuid, + size: 0, // Size will be known after archive creation + date + }); + } + + return result; + } + + async downloadBackup(backupMetadata: BackupFileMetadata): Promise { + // Check if this is a pending backup (multi-file mode) + const pendingBackup = this.pendingBackups.get(backupMetadata.uuid); + + if (pendingBackup) { + // Create the archive now + logger.info(`[rsync] Creating archive for item: ${pendingBackup.itemName}`); + const archive = await createArchiveForTarget( + pendingBackup.target, + pendingBackup.specificPath, + pendingBackup.itemName + ); + + // Remove from pending backups + this.pendingBackups.delete(backupMetadata.uuid); + + return archive.archivePath; + } + + // Default behavior for single backup mode + return backupMetadata.localPath; + } } \ No newline at end of file diff --git a/src/services/backup/rsync/utils.ts b/src/services/backup/rsync/utils.ts index 1b0f4e4..3302324 100644 --- a/src/services/backup/rsync/utils.ts +++ b/src/services/backup/rsync/utils.ts @@ -151,9 +151,12 @@ async function runCommand(command: string, args: string[], options?: CommandOpti }); } -function buildRsyncArgs(target: RsyncTarget, destination: string) { - const args = ["-az"]; +export function isArchiveFile(fileName: string) { + const extensions = [".tar.gz", ".tgz", ".tar"]; + return extensions.some((ext) => fileName.endsWith(ext)); +} +function buildSshCommand(target: RsyncTarget): string[] { const sshParts = ["ssh"]; if (target.port) { @@ -168,6 +171,14 @@ function buildRsyncArgs(target: RsyncTarget, destination: string) { sshParts.push(option); } + return sshParts; +} + +function buildRsyncArgs(target: RsyncTarget, destination: string, specificPath?: string) { + const args = ["-az"]; + + const sshParts = buildSshCommand(target); + for (const exclude of target.excludes) { args.push("--exclude", exclude); } @@ -176,28 +187,76 @@ function buildRsyncArgs(target: RsyncTarget, destination: string) { args.push("-e", sshParts.join(" ")); - const remoteSpec = `${target.user ? `${target.user}@` : ""}${formatHost(target.host)}:${target.path}`; + const remotePath = specificPath || target.path; + const remoteSpec = `${target.user ? `${target.user}@` : ""}${formatHost(target.host)}:${remotePath}`; args.push(remoteSpec, destination); return args; } -export async function createArchiveForTarget(target: RsyncTarget) { - const sanitizedName = sanitizeName(target.name); +export async function listRemoteFiles(target: RsyncTarget, remotePath: string): Promise { + const sshCommand = buildSshCommand(target); + const host = formatHost(target.host); + const remoteHost = target.user ? `${target.user}@${host}` : host; + + const findCommand = `find "${remotePath}" -mindepth 1 -maxdepth 1 -printf '%f\\n'`; + + const [bin, ...sshArgs] = sshCommand; + const args = [...sshArgs, remoteHost, findCommand]; + + return new Promise((resolve, reject) => { + const child = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] }); + + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", (data) => stdout += data.toString()); + child.stderr?.on("data", (data) => stderr += data.toString()); + + child.on("close", (code) => { + if (code === 0) { + const files = stdout.trim().split("\n").filter(l => l.length > 0); + resolve(files); + } else { + reject(new Error(`SSH Exit ${code}: ${stdout} - ${stderr.trim()}`)); + } + }); + }); +} + +export async function createArchiveForTarget(target: RsyncTarget, specificPath?: string, itemName?: string) { + const sanitizedName = itemName ? sanitizeName(itemName) : sanitizeName(target.name); const date = new Date(); const timestamp = buildTimestamp(date); const workingDirectory = join(RSYNC_TMP_ROOT, `${sanitizedName}-${timestamp}`); - const archiveName = `${sanitizedName}-${timestamp}.tar.gz`; - const archivePath = join(RSYNC_TMP_ROOT, archiveName); + let archiveName = `${sanitizedName}-${timestamp}.tar.gz`; + let archivePath = join(RSYNC_TMP_ROOT, archiveName); mkdirSync(workingDirectory, { recursive: true }); + const remotePath = specificPath || target.path; + const displayPath = specificPath ? `${target.path}/${specificPath}` : target.path; + try { - logger.info(`[rsync] Starting sync for "${target.name}" (${target.host}:${target.path}).`); - const rsyncArgs = buildRsyncArgs(target, workingDirectory); + logger.info(`[rsync] Starting sync for "${target.name}" (${target.host}:${displayPath}).`); + const rsyncArgs = buildRsyncArgs(target, workingDirectory, remotePath); await runCommand("rsync", rsyncArgs, { logPrefix: "rsync" }); + if (!isArchiveFile(archiveName)) { + archiveName += ".tar.gz"; + archivePath = join(RSYNC_TMP_ROOT, archiveName); + + return { + archiveName, + archivePath, + sanitizedName: archiveName, + timestamp, + size: statSync(archivePath).size, + date + }; + } + logger.info(`[rsync] Creating archive for "${target.name}".`); await runCommand("tar", ["-czf", archivePath, "-C", workingDirectory, "."], { logPrefix: "tar" }); diff --git a/src/services/files/encryption.ts b/src/services/files/encryption.ts index 2304fec..00fab48 100644 --- a/src/services/files/encryption.ts +++ b/src/services/files/encryption.ts @@ -2,6 +2,7 @@ import "dotenv/config"; import { createMessage, encrypt, readKey } from "openpgp"; import { readFile, writeFile, rename } from "node:fs/promises"; import { createReadStream, createWriteStream } from "node:fs"; +import { Readable, Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { logger } from "../log"; @@ -21,20 +22,25 @@ export async function EncryptFile(filePath: string): Promise { logger.debug(`Encrypting file ${filePath}...`); // Use streaming to handle large files without loading them entirely into memory - const readStream = createReadStream(filePath); + const nodeReadStream = createReadStream(filePath); + // Convert Node.js stream to WebStream (required by openpgp v6+) + const webReadStream = Readable.toWeb(nodeReadStream) as ReadableStream; const tempFilePath = `${filePath}.tmp`; const writeStream = createWriteStream(tempFilePath); const encryptedStream = await encrypt({ - message: await createMessage({ binary: readStream }), + message: await createMessage({ binary: webReadStream }), encryptionKeys: publicKey, format: 'armored' }); logger.debug(`File ${filePath} encrypted successfully, writing to temporary file...`); + // Convert WebStream back to Node.js stream for pipeline + const nodeEncryptedStream = Readable.fromWeb(encryptedStream as import("stream/web").ReadableStream); + // Pipe the encrypted stream to the output file - await pipeline(encryptedStream, writeStream); + await pipeline(nodeEncryptedStream, writeStream); // Replace the original file with the encrypted one await rename(tempFilePath, filePath); diff --git a/src/services/storage/sftp/main.ts b/src/services/storage/sftp/main.ts index d618b7a..01c6934 100644 --- a/src/services/storage/sftp/main.ts +++ b/src/services/storage/sftp/main.ts @@ -15,6 +15,8 @@ const { export class SFTPStorage extends StorageClass { private client: SftpClient; + private keepaliveInterval: number = 5_000; // 5 seconds + private keepaliveCountMax: number = 1_000; // Max keepalive attempts constructor() { super(); @@ -30,6 +32,9 @@ export class SFTPStorage extends StorageClass { host: hostIp, port: SFTP_PORT ? +SFTP_PORT : 22, username: SFTP_USER, + debug: (msg: string) => logger.silly(msg), + keepaliveInterval: this.keepaliveInterval, + keepaliveCountMax: this.keepaliveCountMax }; // Authentication: prefer SSH key over password @@ -55,57 +60,90 @@ export class SFTPStorage extends StorageClass { logger.info(`Connected to SFTP server: ${hostIp}:${config.port}`); } + async disconnect() { + await this.client.end(); + logger.info(`Disconnected from SFTP server`); + } + async deleteFile(filePath: string) { + await this.connect(); + try { await this.client.delete(filePath); logger.debug(`Deleted file: ${filePath}`); } catch (error) { logger.error(`Failed to delete file ${filePath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async uploadFile(filePath: string, destination: string) { + await this.connect(); + try { - await this.client.put(filePath, destination); + await this.client.fastPut(filePath, destination, { + concurrency: 64, + chunkSize: 32768, + step: (totalTransferred, chunk, total) => { + logger.debug(`Upload progress: ${((totalTransferred / total) * 100).toFixed(2)}%`); + } + }) logger.debug(`Uploaded file: ${filePath}, to: ${destination}`); } catch (error) { logger.error(`Failed to upload file ${filePath} to ${destination}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async createFolder(folderPath: string) { + await this.connect(); + try { await this.client.mkdir(folderPath, true); // recursive = true logger.debug(`Created folder: ${folderPath}`); } catch (error) { logger.error(`Failed to create folder ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async deleteFolder(folderPath: string) { + await this.connect(); + try { await this.client.rmdir(folderPath, true); // recursive = true logger.debug(`Deleted folder: ${folderPath}`); } catch (error) { logger.error(`Failed to delete folder ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async folderExists(folderPath: string): Promise { + await this.connect(); + try { const stat = await this.client.stat(folderPath); return stat.isDirectory; } catch (error) { // If stat fails, the folder doesn't exist return false; + } finally { + await this.disconnect(); } } async folderSizeBytes(folderPath: string) { + await this.connect(); + try { const list = await this.client.list(folderPath); let totalSize = 0; @@ -124,10 +162,14 @@ export class SFTPStorage extends StorageClass { } catch (error) { logger.error(`Failed to calculate folder size for ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async listFiles(folderPath: string) { + await this.connect(); + try { const list = await this.client.list(folderPath); @@ -141,14 +183,14 @@ export class SFTPStorage extends StorageClass { } catch (error) { logger.error(`Failed to list files in folder ${folderPath}: ${error}`); throw error; + } finally { + await this.disconnect(); } } async close() { - await this.client.end(); } async init() { - await this.connect(); } }