From 5ef897a2a1bd2937c2bc950a2842cb634c68e541 Mon Sep 17 00:00:00 2001 From: Cameron Banowsky Date: Thu, 5 Feb 2026 14:53:08 -0800 Subject: [PATCH] fix(worker): stream R2 objects on retrieval to support large file downloads Replace arrayBuffer() buffering with direct R2 body streaming for regular paste retrieval. This prevents worker memory exhaustion (Error 1101) when serving multipart-uploaded files that exceed the ~128MB worker memory limit. Markdown rendering still buffers as the marked library requires full content. Also includes compiled CLI output for multipart upload support and enables worker observability. Co-Authored-By: Claude Opus 4.6 --- cli/index.js | 179 +++++++++++++++++++++++++++++++++++++++------- package-lock.json | 4 +- src/index.ts | 12 ++-- wrangler.toml | 5 ++ 4 files changed, 166 insertions(+), 34 deletions(-) diff --git a/cli/index.js b/cli/index.js index f28da52..a3392ae 100755 --- a/cli/index.js +++ b/cli/index.js @@ -33,6 +33,10 @@ catch (error) { } // Import analytics import { analytics } from './analytics.js'; +// Import large file upload modules +import { shouldUseMultipart, uploadLargeFile, resumeUpload, abortUpload, } from './largeFileUpload.js'; +import { LARGE_FILE_CONSTANTS } from '../src/types/index.js'; +import { listResumeStates, formatResumeState, } from './resumeState.js'; // Import our core modules import { generateKeyPair, addFriendKey, listKeys, getKey, removeKey, loadKeyDatabase, saveKeyDatabase, ensureDirectories } from './keyManager.js'; import { encryptContent, decryptContent } from './encryptionUtils.js'; @@ -860,6 +864,12 @@ program .option('--enhanced', 'Use enhanced interactive mode with advanced key selection features') .option('--debug', 'Debug mode: show encrypted content without uploading') .option('-c, --copy', 'Copy the URL to clipboard automatically') + // Large file options + .option('--chunk-size ', 'Chunk size in MB for large file uploads (default: 10, min: 5)') + .option('--resume ', 'Resume an interrupted upload from a saved state file') + .option('--no-progress', 'Disable progress bar for large file uploads') + .option('--list-uploads', 'List interrupted uploads that can be resumed') + .option('--abort-upload ', 'Abort an interrupted upload by session ID') // PGP options .option('--pgp', 'Use PGP encryption instead of hybrid RSA/AES') .option('--pgp-key-file ', 'Use a specific PGP public key file for encryption') @@ -922,6 +932,74 @@ Encryption: } return; } + // ============================================ + // Large File Upload Handlers + // ============================================ + // List interrupted uploads + if (options.listUploads) { + const states = await listResumeStates(); + if (states.length === 0) { + console.log('No interrupted uploads found.'); + } + else { + console.log(`Found ${states.length} interrupted upload(s):\n`); + for (const state of states) { + console.log(formatResumeState(state)); + console.log('---'); + } + console.log('\nTo resume: dedpaste send --resume '); + console.log('To abort: dedpaste send --abort-upload '); + } + return; + } + // Abort an interrupted upload + if (options.abortUpload) { + try { + await abortUpload(options.abortUpload); + console.log('Upload aborted successfully.'); + } + catch (error) { + console.error(`Failed to abort upload: ${error.message}`); + process.exit(1); + } + return; + } + // Resume an interrupted upload + if (options.resume) { + try { + console.log('Resuming interrupted upload...'); + const url = await resumeUpload(options.resume, { + showProgress: options.progress !== false, + }); + // Handle clipboard and output + if (options.copy) { + try { + const cleanUrl = url.trim(); + if (clipboard.default) { + clipboard.default.writeSync(cleanUrl); + } + else { + clipboard.writeSync(cleanUrl); + } + } + catch (error) { + console.error(`Unable to copy to clipboard: ${error.message}`); + } + } + if (options.output) { + console.log(url.trim()); + } + else { + console.log(`\nāœ“ Upload resumed and completed!\n${options.copy ? 'šŸ“‹ URL copied to clipboard: ' : 'šŸ“‹ '} ${url.trim()}\n`); + } + process.exit(0); + } + catch (error) { + console.error(`Failed to resume upload: ${error.message}`); + process.exit(1); + } + return; + } let content; let contentType; let recipientName = options.for; @@ -1075,34 +1153,80 @@ Encryption: try { // Extract filename if uploading a file const filename = options.file ? path.basename(options.file) : ''; - const headers = { - 'Content-Type': contentType, - 'User-Agent': `dedpaste-cli/${packageJson.version}` - }; - // Include filename header if we have a file - if (filename) { - headers['X-Filename'] = filename; + let url; + // Check if we should use multipart upload for large files + // Only use multipart for file uploads (not stdin) and files over threshold + const useMultipart = options.file && + !shouldEncrypt && // Encrypted large files not yet supported in multipart + shouldUseMultipart(options.file); + if (useMultipart) { + // Large file - use multipart upload + console.log(`Large file detected (>${LARGE_FILE_CONSTANTS.MULTIPART_THRESHOLD / (1024 * 1024)}MB). Using multipart upload...`); + // Parse chunk size if provided + let chunkSize; + if (options.chunkSize) { + const chunkMb = parseInt(options.chunkSize, 10); + if (isNaN(chunkMb) || chunkMb < 5) { + console.error('Error: Chunk size must be at least 5 MB'); + process.exit(1); + } + chunkSize = chunkMb * 1024 * 1024; + } + try { + url = await uploadLargeFile(options.file, { + apiUrl: API_URL, + chunkSize, + isOneTime: options.temp, + isEncrypted: false, // Encryption handled separately + showProgress: options.progress !== false, + }); + } + catch (uploadError) { + console.error(`Large file upload failed: ${uploadError.message}`); + console.error('You can resume the upload later with: dedpaste send --resume '); + console.error('To see interrupted uploads: dedpaste send --list-uploads'); + process.exit(1); + } + // Track paste creation for large files + analytics.trackPasteCreated({ + type: options.temp ? 'one_time' : 'regular', + content_type: contentType, + size_bytes: fs.statSync(options.file).size, + encryption_type: 'none', + method: 'file' // Use 'file' as the method type + }); } - const response = await fetch(`${API_URL}${endpoint}`, { - method: 'POST', - headers, - body: content - }); - if (!response.ok) { - console.error(`Error: ${response.status} ${response.statusText}`); - const errorText = await response.text(); - console.error(errorText); - process.exit(1); + else { + // Standard upload (small files or stdin) + const headers = { + 'Content-Type': contentType, + 'User-Agent': `dedpaste-cli/${packageJson.version}` + }; + // Include filename header if we have a file + if (filename) { + headers['X-Filename'] = filename; + } + const response = await fetch(`${API_URL}${endpoint}`, { + method: 'POST', + headers, + body: content + }); + if (!response.ok) { + console.error(`Error: ${response.status} ${response.statusText}`); + const errorText = await response.text(); + console.error(errorText); + process.exit(1); + } + url = await response.text(); + // Track paste creation + analytics.trackPasteCreated({ + type: options.temp ? 'one_time' : 'regular', + content_type: contentType, + size_bytes: content.length, + encryption_type: options.encrypt ? 'RSA' : 'none', + method: options.file ? 'file' : 'stdin' + }); } - const url = await response.text(); - // Track paste creation - analytics.trackPasteCreated({ - type: options.temp ? 'one_time' : 'regular', - content_type: contentType, - size_bytes: content.length, - encryption_type: options.encrypt ? 'RSA' : 'none', - method: options.file ? 'file' : 'stdin' - }); // Copy to clipboard if requested if (options.copy) { try { @@ -1132,8 +1256,9 @@ Encryption: encryptionMessage = 'šŸ”’ This paste is encrypted and can only be decrypted with your private key\n'; } } + const isLargeFile = useMultipart; console.log(` -āœ“ Paste created successfully! +āœ“ Paste created successfully!${isLargeFile ? ' (multipart upload)' : ''} ${options.temp ? 'āš ļø This is a one-time paste that will be deleted after first view\n' : ''} ${encryptionMessage} ${options.copy ? 'šŸ“‹ URL copied to clipboard: ' : 'šŸ“‹ '} ${url.trim()} diff --git a/package-lock.json b/package-lock.json index 63e5b04..1b011f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dedpaste", - "version": "1.21.3", + "version": "1.22.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dedpaste", - "version": "1.21.3", + "version": "1.22.0", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/src/index.ts b/src/index.ts index fc0a66e..4b19f40 100644 --- a/src/index.ts +++ b/src/index.ts @@ -975,8 +975,7 @@ async function handleGet( return new Response("Paste not found", { status: 404 }); } - // Regular paste - get the content and metadata - const content = await paste.arrayBuffer(); + // Regular paste - get metadata (available without reading the body) let contentType = "text/plain"; let filename = ""; @@ -1014,7 +1013,8 @@ async function handleGet( filename.endsWith(".markdown"); if (isMarkdown && !isEncrypted && !wantsRaw) { - // Convert markdown to HTML for browser viewing + // Markdown rendering requires buffering the content into memory + const content = await paste.arrayBuffer(); const textContent = new TextDecoder().decode(content); const renderedHTML = await renderMarkdownAsHTML(textContent, id, filename); @@ -1036,11 +1036,13 @@ async function handleGet( const isViewableInBrowser = isViewableContentType(contentType); const disposition = isViewableInBrowser ? "inline" : "attachment"; - // Return the paste content with robust caching headers - return new Response(content, { + // Stream the R2 object body directly to avoid buffering large files in memory. + // This is critical for multipart-uploaded files which can be up to 5GB. + return new Response(paste.body, { headers: { "Content-Type": contentType, "Content-Disposition": `${disposition}; filename="${effectiveFilename}"`, + "Content-Length": paste.size.toString(), "Access-Control-Allow-Origin": "*", "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate", Pragma: "no-cache", diff --git a/wrangler.toml b/wrangler.toml index 361137f..e12b92c 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -26,3 +26,8 @@ preview_id = "0d25f4b9e61a44ab92634e7941cea0a0" binding = "UPLOAD_SESSIONS" id = "c0251d912bb045df9c363b1b00be81a1" preview_id = "3e505a5b19a14816ba1105c7772a2a4c" + +[observability] +enabled = true +head_sampling_rate = 1 +