From e0371b8620d247f8aec75041898044c0476aee66 Mon Sep 17 00:00:00 2001 From: harsha Date: Thu, 24 Apr 2025 08:51:44 +0530 Subject: [PATCH 1/4] added local script to test, with SNS topic --- index.js | 191 +++++++++++++++++++++++++++++++++------------- package-lock.json | 15 +++- package.json | 3 +- test-local.js | 11 +++ 4 files changed, 167 insertions(+), 53 deletions(-) create mode 100644 test-local.js diff --git a/index.js b/index.js index a8f0c4f..5901f85 100755 --- a/index.js +++ b/index.js @@ -1,122 +1,211 @@ -'use strict'; +"use strict"; -const util = require('node:util'); -const exec = util.promisify(require('node:child_process').exec); -const AWS = require('aws-sdk'); -const AdmZip = require('adm-zip'); -const dayjs = require('dayjs'); -const axios = require('axios'); +const util = require("node:util"); +const exec = util.promisify(require("node:child_process").exec); +const AWS = require("aws-sdk"); +const AdmZip = require("adm-zip"); +const dayjs = require("dayjs"); +const axios = require("axios"); // ENVIRONMENT VARIABLES const dumpOptions = process.env.MONGODUMP_OPTIONS; const bucketName = process.env.S3_BUCKET; const s3bucket = new AWS.S3({ params: { Bucket: bucketName } }); -const s3StorageClass = process.env.S3_STORAGE_CLASS || 'STANDARD'; -const zipFilename = process.env.ZIP_FILENAME || 'mongodb_backup'; -const folderPrefix = process.env.FOLDER_PREFIX || 'mongodb_backups'; -const dateFormat = process.env.DATE_FORMAT || 'YYYYMMDD_HHmmss'; -const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL; -const environment = process.env.ENVIRONMENT || 'unknown'; +const s3StorageClass = process.env.S3_STORAGE_CLASS || "STANDARD"; +const zipFilename = process.env.ZIP_FILENAME || "mongodb_backup"; +const folderPrefix = process.env.FOLDER_PREFIX || "mongodb_backups"; +const dateFormat = process.env.DATE_FORMAT || "YYYYMMDD_HHmmss"; +const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL; +const environment = process.env.ENVIRONMENT || "unknown"; const backupsToRetain = parseInt(process.env.BACKUPS_TO_RETAIN) || 10; +// SNS setup +const sns = new AWS.SNS({ region: process.env.AWS_REGION || "us-east-1" }); +const snsTopicArn = process.env.SNS_TOPIC_ARN; // Function to send notifications to Slack const notifySlack = async (message) => { if (!slackWebhookUrl) { - console.warn('Slack webhook URL is not set'); + console.warn("Slack webhook URL is not set"); return; } try { - await axios.post(slackWebhookUrl, { text: `[${environment.toUpperCase()}] ${message}` }); + await axios.post(slackWebhookUrl, { + text: `[${environment.toUpperCase()}] ${message}`, + }); } catch (err) { - console.error('Failed to send Slack notification:', err); + console.error("Failed to send Slack notification:", err); + } +}; + +// Helper to send SNS notifications +const notifySNS = async (subject, message) => { + if (!snsTopicArn) { + console.warn("SNS topic ARN is not set"); + return; + } + try { + await sns + .publish({ + TopicArn: snsTopicArn, + Subject: subject, + Message: message, + }) + .promise(); + } catch (err) { + console.error("Failed to send SNS notification:", err); } }; // Backup process exports.handler = async function (_event, _context) { - console.info(`[${environment.toUpperCase()}] MongoDB backup to S3 bucket '${bucketName}' is starting`); + console.info( + `[${environment.toUpperCase()}] MongoDB backup to S3 bucket '${bucketName}' is starting` + ); - process.env['PATH'] = process.env['PATH'] + ':' + process.env['LAMBDA_TASK_ROOT']; + process.env["PATH"] = + process.env["PATH"] + ":" + process.env["LAMBDA_TASK_ROOT"]; - const fileName = zipFilename + '_' + dayjs().format(dateFormat); + const fileName = zipFilename + "_" + dayjs().format(dateFormat); const folderName = `/tmp/${fileName}/`; let zipBuffer = null; try { - console.info(`[${environment.toUpperCase()}] Creating directory: ${folderName}`); + console.info( + `[${environment.toUpperCase()}] Creating directory: ${folderName}` + ); await exec(`mkdir -p ${folderName}`); } catch (err) { - console.error(`[${environment.toUpperCase()}] Failed to create directory ${folderName}`, err); - await notifySlack(`Failed to create directory ${folderName}: ${err.message}`); + console.error( + `[${environment.toUpperCase()}] Failed to create directory ${folderName}`, + err + ); + await notifySlack( + `Failed to create directory ${folderName}: ${err.message}` + ); + await notifySNS( + "MongoDB Backup Failed", + `Failed to create directory ${folderName}: ${err.message}` + ); throw new Error(`Failed to create directory ${folderName}: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Executing mongodump with options: ${dumpOptions}`); - const { stdout, stderr } = await exec(`mongodump ${dumpOptions} --out ${folderName}`); + console.info( + `[${environment.toUpperCase()}] Executing mongodump with options: ${dumpOptions}` + ); + const { stdout, stderr } = await exec( + `mongodump ${dumpOptions} --out ${folderName}` + ); console.info(`[${environment.toUpperCase()}] mongodump stdout:`, stdout); console.error(`[${environment.toUpperCase()}] mongodump stderr:`, stderr); } catch (err) { - console.error(`[${environment.toUpperCase()}] mongodump command failed:`, err); + console.error( + `[${environment.toUpperCase()}] mongodump command failed:`, + err + ); await notifySlack(`mongodump command failed: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `mongodump command failed: ${err.message}` + ); throw new Error(`mongodump command failed: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Creating ZIP archive from folder: ${folderName}`); + console.info( + `[${environment.toUpperCase()}] Creating ZIP archive from folder: ${folderName}` + ); const zip = new AdmZip(); zip.addLocalFolder(folderName); zipBuffer = zip.toBuffer(); } catch (err) { - console.error(`[${environment.toUpperCase()}] Archive creation failed:`, err); + console.error( + `[${environment.toUpperCase()}] Archive creation failed:`, + err + ); await notifySlack(`Archive creation failed: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `Archive creation failed: ${err.message}` + ); throw new Error(`Archive creation failed: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Uploading ZIP archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.zip`); - await s3bucket.upload({ - Key: `${folderPrefix}/${fileName}.zip`, - Body: zipBuffer, - ContentType: 'application/zip', - ServerSideEncryption: 'AES256', - StorageClass: s3StorageClass - }).promise(); + console.info( + `[${environment.toUpperCase()}] Uploading ZIP archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.zip` + ); + await s3bucket + .upload({ + Key: `${folderPrefix}/${fileName}.zip`, + Body: zipBuffer, + ContentType: "application/zip", + ServerSideEncryption: "AES256", + StorageClass: s3StorageClass, + }) + .promise(); } catch (err) { console.error(`[${environment.toUpperCase()}] Upload to S3 failed:`, err); await notifySlack(`Upload to S3 failed: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `Upload to S3 failed: ${err.message}` + ); throw new Error(`Upload to S3 failed: ${err.message}`); } try { - console.info(`[${environment.toUpperCase()}] Listing objects in S3 bucket: ${bucketName}, Prefix: ${folderPrefix}/`); - const listedObjects = await s3bucket.listObjectsV2({ - Bucket: bucketName, - Prefix: folderPrefix + '/' - }).promise(); + console.info( + `[${environment.toUpperCase()}] Listing objects in S3 bucket: ${bucketName}, Prefix: ${folderPrefix}/` + ); + const listedObjects = await s3bucket + .listObjectsV2({ + Bucket: bucketName, + Prefix: folderPrefix + "/", + }) + .promise(); - const sortedObjects = listedObjects.Contents.sort((a, b) => new Date(b.LastModified) - new Date(a.LastModified)); + const sortedObjects = listedObjects.Contents.sort( + (a, b) => new Date(b.LastModified) - new Date(a.LastModified) + ); const objectsToDelete = sortedObjects.slice(backupsToRetain); if (objectsToDelete.length > 0) { - console.info(`[${environment.toUpperCase()}] Deleting ${objectsToDelete.length} old backups from S3 bucket: ${bucketName}`); - await s3bucket.deleteObjects({ - Bucket: bucketName, - Delete: { - Objects: objectsToDelete.map(obj => ({ Key: obj.Key })) - } - }).promise(); + console.info( + `[${environment.toUpperCase()}] Deleting ${ + objectsToDelete.length + } old backups from S3 bucket: ${bucketName}` + ); + await s3bucket + .deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: objectsToDelete.map((obj) => ({ Key: obj.Key })), + }, + }) + .promise(); } else { console.info(`[${environment.toUpperCase()}] No old backups to delete.`); } } catch (err) { - console.error(`[${environment.toUpperCase()}] Failed to list or delete old backups:`, err); + console.error( + `[${environment.toUpperCase()}] Failed to list or delete old backups:`, + err + ); await notifySlack(`Failed to list or delete old backups: ${err.message}`); + await notifySNS( + "MongoDB Backup Failed", + `Failed to list or delete old backups: ${err.message}` + ); throw new Error(`Failed to list or delete old backups: ${err.message}`); } console.info(`[${environment.toUpperCase()}] Backup completed successfully`); - await notifySlack('Backup completed successfully'); -}; \ No newline at end of file + await notifySlack("Backup completed successfully"); + await notifySNS( + "MongoDB Backup Success", + `Backup completed successfully and uploaded to S3 bucket: ${bucketName}` + ); +}; diff --git a/package-lock.json b/package-lock.json index a93a3d8..e614503 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,8 @@ "adm-zip": "^0.5.10", "aws-sdk": "^2.1423.0", "axios": "^1.7.2", - "dayjs": "^1.11.9" + "dayjs": "^1.11.9", + "dotenv": "^16.5.0" } }, "node_modules/adm-zip": { @@ -172,6 +173,18 @@ "node": ">=0.4.0" } }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/es-define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", diff --git a/package.json b/package.json index 7f3c16e..b493662 100755 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "adm-zip": "^0.5.10", "aws-sdk": "^2.1423.0", "axios": "^1.7.2", - "dayjs": "^1.11.9" + "dayjs": "^1.11.9", + "dotenv": "^16.5.0" }, "scripts": { "test": "echo \"Error: no test specified\" && exit 1" diff --git a/test-local.js b/test-local.js new file mode 100644 index 0000000..5120d58 --- /dev/null +++ b/test-local.js @@ -0,0 +1,11 @@ +require("dotenv").config(); +const { handler } = require("./index"); + +(async () => { + try { + await handler({}, {}); + console.log("Backup completed."); + } catch (err) { + console.error("Backup failed:", err); + } +})(); From 4ea8b5eb24a4f478b094ed4143b61d0fe5e0c861 Mon Sep 17 00:00:00 2001 From: harsha Date: Thu, 24 Apr 2025 09:00:49 +0530 Subject: [PATCH 2/4] update readme --- README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/README.md b/README.md index f4ac276..8e69148 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,49 @@ A notification will be sent each time a backup request is triggered, informing y | ENVIRONMENT | Specify your app environment for custom Slack notifications | No. Default is `unknown` | | SLACK_WEBHOOK_URL | Your Slack webhook URL | No | | BACKUPS_TO_RETAIN | Number of backups to retain in S3. Older backups will be deleted. | No. Default is `10` | +| SNS_TOPIC_ARN | SNS topic ARN for notifications on backup success/failure | No | +| AWS_REGION | AWS region for SDK operations (default: `us-east-1`) | No | + +## SNS Notification Support + +If you wish to receive notifications (email, SMS, etc.) on backup success or failure, configure an [AWS SNS topic](https://docs.aws.amazon.com/sns/latest/dg/sns-create-topic.html) and set the following environment variables: + +```env +SNS_TOPIC_ARN=arn:aws:sns:us-east-1:xxxxxxxxxxxx:mongodb-backup-events +AWS_REGION=us-east-1 +``` + +- Subscribe your email or endpoint to the SNS topic and confirm the subscription. +- The Lambda will send notifications for both success and failure events. + +## Local Testing + +A script named `test-local.js` is provided to allow local execution and debugging of the backup logic. + +### Usage + +1. Ensure your `.env` file is correctly configured with all required variables. +2. Run the following command: + ```bash + node test-local.js + ``` +3. The script will perform a backup using your local MongoDB (or the URI you provide), upload to S3, and send notifications as configured. + +## Example `.env` + +```env +MONGODUMP_OPTIONS=--uri="mongodb://localhost:27017" +S3_BUCKET=d33-ot-db-backups +S3_STORAGE_CLASS=STANDARD +ZIP_FILENAME=mongodb_backup +FOLDER_PREFIX=mongodb_backups +DATE_FORMAT=YYYYMMDD_HHmmss +SLACK_WEBHOOK_URL= +ENVIRONMENT=local +BACKUPS_TO_RETAIN=10 +SNS_TOPIC_ARN=arn:aws:sns:us-east-1:xxxxxxxxxxxx:mongodb-backup-events +AWS_REGION=us-east-1 +``` ## Et voila ! From d92a886bffe11a0e8e7774c5b35a81b316c097da Mon Sep 17 00:00:00 2001 From: MagiciansMac Date: Sat, 14 Feb 2026 08:08:13 +0530 Subject: [PATCH 3/4] Refactor archive creation to use TAR.GZ format instead of ZIP, updating S3 upload accordingly. --- index.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 5901f85..62625bd 100755 --- a/index.js +++ b/index.js @@ -1,9 +1,9 @@ "use strict"; const util = require("node:util"); +const fs = require("node:fs"); const exec = util.promisify(require("node:child_process").exec); const AWS = require("aws-sdk"); -const AdmZip = require("adm-zip"); const dayjs = require("dayjs"); const axios = require("axios"); @@ -114,12 +114,15 @@ exports.handler = async function (_event, _context) { } try { + const archivePath = `/tmp/${fileName}.tar.gz`; console.info( - `[${environment.toUpperCase()}] Creating ZIP archive from folder: ${folderName}` + `[${environment.toUpperCase()}] Creating TAR.GZ archive from folder: ${folderName} -> ${archivePath}` ); - const zip = new AdmZip(); - zip.addLocalFolder(folderName); - zipBuffer = zip.toBuffer(); + + // Use system tar instead of adm-zip to avoid EFAULT read errors in Lambda + await exec(`cd /tmp && tar -czf ${archivePath} ${fileName}`); + + zipBuffer = fs.readFileSync(archivePath); } catch (err) { console.error( `[${environment.toUpperCase()}] Archive creation failed:`, @@ -135,13 +138,13 @@ exports.handler = async function (_event, _context) { try { console.info( - `[${environment.toUpperCase()}] Uploading ZIP archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.zip` + `[${environment.toUpperCase()}] Uploading archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.tar.gz` ); await s3bucket .upload({ - Key: `${folderPrefix}/${fileName}.zip`, + Key: `${folderPrefix}/${fileName}.tar.gz`, Body: zipBuffer, - ContentType: "application/zip", + ContentType: "application/gzip", ServerSideEncryption: "AES256", StorageClass: s3StorageClass, }) From 70b1c0250e5f07434ed740c71ba8477f4b914ea0 Mon Sep 17 00:00:00 2001 From: MagiciansMac Date: Mon, 16 Feb 2026 09:27:33 +0530 Subject: [PATCH 4/4] feat: Switch from tar.gz to AdmZip for creating zip archives for S3 uploads. --- index.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 62625bd..ee27333 100755 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ const exec = util.promisify(require("node:child_process").exec); const AWS = require("aws-sdk"); const dayjs = require("dayjs"); const axios = require("axios"); +const AdmZip = require("adm-zip"); // ENVIRONMENT VARIABLES const dumpOptions = process.env.MONGODUMP_OPTIONS; @@ -114,13 +115,14 @@ exports.handler = async function (_event, _context) { } try { - const archivePath = `/tmp/${fileName}.tar.gz`; + const archivePath = `/tmp/${fileName}.zip`; console.info( - `[${environment.toUpperCase()}] Creating TAR.GZ archive from folder: ${folderName} -> ${archivePath}` + `[${environment.toUpperCase()}] Creating ZIP archive from folder: ${folderName} -> ${archivePath}` ); - // Use system tar instead of adm-zip to avoid EFAULT read errors in Lambda - await exec(`cd /tmp && tar -czf ${archivePath} ${fileName}`); + const zip = new AdmZip(); + zip.addLocalFolder(folderName, fileName); + zip.writeZip(archivePath); zipBuffer = fs.readFileSync(archivePath); } catch (err) { @@ -138,13 +140,13 @@ exports.handler = async function (_event, _context) { try { console.info( - `[${environment.toUpperCase()}] Uploading archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.tar.gz` + `[${environment.toUpperCase()}] Uploading archive to S3 bucket: ${bucketName}, Key: ${folderPrefix}/${fileName}.zip` ); await s3bucket .upload({ - Key: `${folderPrefix}/${fileName}.tar.gz`, + Key: `${folderPrefix}/${fileName}.zip`, Body: zipBuffer, - ContentType: "application/gzip", + ContentType: "application/zip", ServerSideEncryption: "AES256", StorageClass: s3StorageClass, })