diff --git a/README.md b/README.md index f4ac276..8e69148 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,49 @@ A notification will be sent each time a backup request is triggered, informing y | ENVIRONMENT | Specify your app environment for custom Slack notifications | No. Default is `unknown` | | SLACK_WEBHOOK_URL | Your Slack webhook URL | No | | BACKUPS_TO_RETAIN | Number of backups to retain in S3. Older backups will be deleted. | No. Default is `10` | +| SNS_TOPIC_ARN | SNS topic ARN for notifications on backup success/failure | No | +| AWS_REGION | AWS region for SDK operations (default: `us-east-1`) | No | + +## SNS Notification Support + +If you wish to receive notifications (email, SMS, etc.) on backup success or failure, configure an [AWS SNS topic](https://docs.aws.amazon.com/sns/latest/dg/sns-create-topic.html) and set the following environment variables: + +```env +SNS_TOPIC_ARN=arn:aws:sns:us-east-1:xxxxxxxxxxxx:mongodb-backup-events +AWS_REGION=us-east-1 +``` + +- Subscribe your email or endpoint to the SNS topic and confirm the subscription. +- The Lambda will send notifications for both success and failure events. + +## Local Testing + +A script named `test-local.js` is provided to allow local execution and debugging of the backup logic. + +### Usage + +1. Ensure your `.env` file is correctly configured with all required variables. +2. Run the following command: + ```bash + node test-local.js + ``` +3. The script will perform a backup using your local MongoDB (or the URI you provide), upload to S3, and send notifications as configured. + +## Example `.env` + +```env +MONGODUMP_OPTIONS=--uri="mongodb://localhost:27017" +S3_BUCKET=d33-ot-db-backups +S3_STORAGE_CLASS=STANDARD +ZIP_FILENAME=mongodb_backup +FOLDER_PREFIX=mongodb_backups +DATE_FORMAT=YYYYMMDD_HHmmss +SLACK_WEBHOOK_URL= +ENVIRONMENT=local +BACKUPS_TO_RETAIN=10 +SNS_TOPIC_ARN=arn:aws:sns:us-east-1:xxxxxxxxxxxx:mongodb-backup-events +AWS_REGION=us-east-1 +``` ## Et voila ! diff --git a/index.js b/index.js index a8f0c4f..ee27333 100755 --- a/index.js +++ b/index.js @@ -1,122 +1,216 @@ -'use strict'; +"use strict"; -const util = require('node:util'); -const exec = util.promisify(require('node:child_process').exec); -const AWS = require('aws-sdk'); -const AdmZip = require('adm-zip'); -const dayjs = require('dayjs'); -const axios = require('axios'); +const util = require("node:util"); +const fs = require("node:fs"); +const exec = util.promisify(require("node:child_process").exec); +const AWS = require("aws-sdk"); +const dayjs = require("dayjs"); +const axios = require("axios"); +const AdmZip = require("adm-zip"); // ENVIRONMENT VARIABLES const dumpOptions = process.env.MONGODUMP_OPTIONS; const bucketName = process.env.S3_BUCKET; const s3bucket = new AWS.S3({ params: { Bucket: bucketName } }); -const s3StorageClass = process.env.S3_STORAGE_CLASS || 'STANDARD'; -const zipFilename = process.env.ZIP_FILENAME || 'mongodb_backup'; -const folderPrefix = process.env.FOLDER_PREFIX || 'mongodb_backups'; -const dateFormat = process.env.DATE_FORMAT || 'YYYYMMDD_HHmmss'; -const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL; -const environment = process.env.ENVIRONMENT || 'unknown'; +const s3StorageClass = process.env.S3_STORAGE_CLASS || "STANDARD"; +const zipFilename = process.env.ZIP_FILENAME || "mongodb_backup"; +const folderPrefix = process.env.FOLDER_PREFIX || "mongodb_backups"; +const dateFormat = process.env.DATE_FORMAT || "YYYYMMDD_HHmmss"; +const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL; +const environment = process.env.ENVIRONMENT || "unknown"; const backupsToRetain = parseInt(process.env.BACKUPS_TO_RETAIN) || 10; +// SNS setup +const sns = new AWS.SNS({ region: process.env.AWS_REGION || "us-east-1" }); +const snsTopicArn = process.env.SNS_TOPIC_ARN; // Function to send notifications to Slack const notifySlack = async (message) => { if (!slackWebhookUrl) { - console.warn('Slack webhook URL is not set'); + console.warn("Slack webhook URL is not set"); return; } try { - await axios.post(slackWebhookUrl, { text: `[${environment.toUpperCase()}] ${message}` }); + await axios.post(slackWebhookUrl, { + text: `[${environment.toUpperCase()}] ${message}`, + }); } catch (err) { - console.error('Failed to send Slack notification:', err); + console.error("Failed to send Slack notification:", err); + } +}; + +// Helper to send SNS notifications +const notifySNS = async (subject, message) => { + if (!snsTopicArn) { + console.warn("SNS topic ARN is not set"); + return; + } + try { + await sns + .publish({ + TopicArn: snsTopicArn, + Subject: subject, + Message: message, + }) + .promise(); + } catch (err) { + console.error("Failed to send SNS notification:", err); } }; // Backup process exports.handler = async function (_event, _context) { - console.info(`[${environment.toUpperCase()}] MongoDB backup to S3 bucket '${bucketName}' is starting`); + console.info( + `[${environment.toUpperCase()}] MongoDB backup to S3 bucket '${bucketName}' is starting` + ); - process.env['PATH'] = process.env['PATH'] + ':' + process.env['LAMBDA_TASK_ROOT']; + process.env["PATH"] = + process.env["PATH"] + ":" + process.env["LAMBDA_TASK_ROOT"]; - const fileName = zipFilename + '_' + dayjs().format(dateFormat); + const fileName = zipFilename + "_" + dayjs().format(dateFormat); const folderName = `/tmp/${fileName}/`; let zipBuffer = null; try { - console.info(`[${environment.toUpperCase()}] Creating directory: ${folderName}`); + console.info( + `[${environment.toUpperCase()}] Creating directory: ${folderName}` + ); await exec(`mkdir -p ${folderName}`); } catch (err) { - console.error(`[${environment.toUpperCase()}] Failed to create directory ${folderName}`, err); - await notifySlack(`Failed to create directory ${folderName}: ${err.message}`); + console.error( + `[${environment.toUpperCase()}] Failed to create directory ${folderName}`, + err + ); + await notifySlack( + `Failed to create directory ${folderName}: ${err.message}` + ); + await notifySNS( + "MongoDB Backup Failed", + `Failed to create directory ${folderName}: ${err.message}` + ); throw new Error(`Failed to create directory ${folderName}: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Executing mongodump with options: ${dumpOptions}`); - const { stdout, stderr } = await exec(`mongodump ${dumpOptions} --out ${folderName}`); + console.info( + `[${environment.toUpperCase()}] Executing mongodump with options: ${dumpOptions}` + ); + const { stdout, stderr } = await exec( + `mongodump ${dumpOptions} --out ${folderName}` + ); console.info(`[${environment.toUpperCase()}] mongodump stdout:`, stdout); console.error(`[${environment.toUpperCase()}] mongodump stderr:`, stderr); } catch (err) { - console.error(`[${environment.toUpperCase()}] mongodump command failed:`, err); + console.error( + `[${environment.toUpperCase()}] mongodump command failed:`, + err + ); await notifySlack(`mongodump command failed: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `mongodump command failed: ${err.message}` + ); throw new Error(`mongodump command failed: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Creating ZIP archive from folder: ${folderName}`); + const archivePath = `/tmp/${fileName}.zip`; + console.info( + `[${environment.toUpperCase()}] Creating ZIP archive from folder: ${folderName} -> ${archivePath}` + ); + const zip = new AdmZip(); - zip.addLocalFolder(folderName); - zipBuffer = zip.toBuffer(); + zip.addLocalFolder(folderName, fileName); + zip.writeZip(archivePath); + + zipBuffer = fs.readFileSync(archivePath); } catch (err) { - console.error(`[${environment.toUpperCase()}] Archive creation failed:`, err); + console.error( + `[${environment.toUpperCase()}] Archive creation failed:`, + err + ); await notifySlack(`Archive creation failed: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `Archive creation failed: ${err.message}` + ); throw new Error(`Archive creation failed: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Uploading ZIP archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.zip`); - await s3bucket.upload({ - Key: `${folderPrefix}/${fileName}.zip`, - Body: zipBuffer, - ContentType: 'application/zip', - ServerSideEncryption: 'AES256', - StorageClass: s3StorageClass - }).promise(); + console.info( + `[${environment.toUpperCase()}] Uploading archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.zip` + ); + await s3bucket + .upload({ + Key: `${folderPrefix}/${fileName}.zip`, + Body: zipBuffer, + ContentType: "application/zip", + ServerSideEncryption: "AES256", + StorageClass: s3StorageClass, + }) + .promise(); } catch (err) { console.error(`[${environment.toUpperCase()}] Upload to S3 failed:`, err); await notifySlack(`Upload to S3 failed: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `Upload to S3 failed: ${err.message}` + ); throw new Error(`Upload to S3 failed: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Listing objects in S3 bucket: ${bucketName}, Prefix: ${folderPrefix}/`); - const listedObjects = await s3bucket.listObjectsV2({ - Bucket: bucketName, - Prefix: folderPrefix + '/' - }).promise(); + console.info( + `[${environment.toUpperCase()}] Listing objects in S3 bucket: ${bucketName}, Prefix: ${folderPrefix}/` + ); + const listedObjects = await s3bucket + .listObjectsV2({ + Bucket: bucketName, + Prefix: folderPrefix + "/", + }) + .promise(); - const sortedObjects = listedObjects.Contents.sort((a, b) => new Date(b.LastModified) - new Date(a.LastModified)); + const sortedObjects = listedObjects.Contents.sort( + (a, b) => new Date(b.LastModified) - new Date(a.LastModified) + ); const objectsToDelete = sortedObjects.slice(backupsToRetain); if (objectsToDelete.length > 0) { - console.info(`[${environment.toUpperCase()}] Deleting ${objectsToDelete.length} old backups from S3 bucket: ${bucketName}`); - await s3bucket.deleteObjects({ - Bucket: bucketName, - Delete: { - Objects: objectsToDelete.map(obj => ({ Key: obj.Key })) - } - }).promise(); + console.info( + `[${environment.toUpperCase()}] Deleting ${ + objectsToDelete.length + } old backups from S3 bucket: ${bucketName}` + ); + await s3bucket + .deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: objectsToDelete.map((obj) => ({ Key: obj.Key })), + }, + }) + .promise(); } else { console.info(`[${environment.toUpperCase()}] No old backups to delete.`); } } catch (err) { - console.error(`[${environment.toUpperCase()}] Failed to list or delete old backups:`, err); + console.error( + `[${environment.toUpperCase()}] Failed to list or delete old backups:`, + err + ); await notifySlack(`Failed to list or delete old backups: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `Failed to list or delete old backups: ${err.message}` + ); throw new Error(`Failed to list or delete old backups: ${err.message}`); } console.info(`[${environment.toUpperCase()}] Backup completed successfully`); - await notifySlack('Backup completed successfully'); -}; \ No newline at end of file + await notifySlack("Backup completed successfully"); + await notifySNS( + "MongoDB Backup Success", + `Backup completed successfully and uploaded to S3 bucket: ${bucketName}` + ); +}; diff --git a/package-lock.json b/package-lock.json index a93a3d8..e614503 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,8 @@ "adm-zip": "^0.5.10", "aws-sdk": "^2.1423.0", "axios": "^1.7.2", - "dayjs": "^1.11.9" + "dayjs": "^1.11.9", + "dotenv": "^16.5.0" } }, "node_modules/adm-zip": { @@ -172,6 +173,18 @@ "node": ">=0.4.0" } }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/es-define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", diff --git a/package.json b/package.json index 7f3c16e..b493662 100755 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "adm-zip": "^0.5.10", "aws-sdk": "^2.1423.0", "axios": "^1.7.2", - "dayjs": "^1.11.9" + "dayjs": "^1.11.9", + "dotenv": "^16.5.0" }, "scripts": { "test": "echo \"Error: no test specified\" && exit 1" diff --git a/test-local.js b/test-local.js new file mode 100644 index 0000000..5120d58 --- /dev/null +++ b/test-local.js @@ -0,0 +1,11 @@ +require("dotenv").config(); +const { handler } = require("./index"); + +(async () => { + try { + await handler({}, {}); + console.log("Backup completed."); + } catch (err) { + console.error("Backup failed:", err); + } +})();