diff --git a/package.json b/package.json index fb6e603..e936f7c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@internxt/inxt-js", "author": "Internxt ", - "version": "2.3.1", + "version": "3.0.0", "description": "", "main": "build/index.js", "types": "build/index.d.ts", @@ -45,7 +45,7 @@ }, "dependencies": { "@internxt/lib": "1.4.1", - "@internxt/sdk": "1.15.1", + "@internxt/sdk": "1.15.2", "async": "3.2.6", "axios": "1.13.5", "bip39": "3.1.0", diff --git a/src/index.ts b/src/index.ts index eb43e77..20297d2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,7 @@ import { Bridge, CreateFileTokenResponse, GetDownloadLinksResponse } from './ser import { HashStream } from './lib/utils/streams'; import { downloadFileV2 } from './lib/core/download/downloadV2'; import { FileVersionOneError } from '@internxt/sdk/dist/network/download'; -import { uploadFileMultipart, uploadFileV2 } from './lib/core/upload/uploadV2'; +import { upload as uploadFileV2 } from './lib/core/upload/uploadV2'; type GetBucketsCallback = (err: Error | null, result: any) => void; @@ -150,82 +150,20 @@ export class Environment { this.config.encryptionKey = newEncryptionKey; } - uploadMultipartFile(bucketId: string, opts: UploadOptions): ActionState { - const uploadState = new ActionState(ActionTypes.Upload); - + upload: UploadStrategyFunction = async (bucketId: string, opts: UploadOptions) => { if (!this.config.encryptionKey) { - opts.finishedCallback(Error('Mnemonic was not provided, please, provide a mnemonic'), null); - - return uploadState; + throw Error('Mnemonic was not provided, please, provide a mnemonic'); } if (!this.config.bridgeUrl) { - opts.finishedCallback(Error('Missing param "bridgeUrl"'), null); - - return uploadState; + throw Error('Missing param "bridgeUrl"'); } if (!bucketId) { - opts.finishedCallback(Error('Bucket id was not provided'), null); - - return uploadState; + throw Error('Bucket id was not provided'); } - // if (!opts.parts || isNaN(opts.parts) || opts.parts < 2) { - // opts.finishedCallback(Error('Invalid "parts" parameter. Expected number > 1'), null); - - // return uploadState; - // } - - uploadFileMultipart( - opts.fileSize, - opts.source, - bucketId, - this.config.encryptionKey, - this.config.bridgeUrl, - { - user: this.config.bridgeUser, - pass: this.config.bridgePass, - }, - opts.progressCallback, - uploadState, - this.config.appDetails, - ) - .then((fileId) => { - opts.finishedCallback(null, fileId); - }) - .catch((err) => { - opts.finishedCallback( - err.message === 'The operation was aborted' ? new Error('Process killed by user') : err, - null, - ); - }); - - return uploadState; - } - - upload: UploadStrategyFunction = (bucketId: string, opts: UploadOptions) => { - const uploadState = new ActionState(ActionTypes.Upload); - - if (!this.config.encryptionKey) { - opts.finishedCallback(Error('Mnemonic was not provided, please, provide a mnemonic'), null); - - return uploadState; - } - - if (!this.config.bridgeUrl) { - opts.finishedCallback(Error('Missing param "bridgeUrl"'), null); - - return uploadState; - } - - if (!bucketId) { - opts.finishedCallback(Error('Bucket id was not provided'), null); - - return uploadState; - } - - uploadFileV2( + return await uploadFileV2( opts.fileSize, opts.source, bucketId, @@ -237,19 +175,8 @@ export class Environment { }, this.config.appDetails, opts.progressCallback, - uploadState, - ) - .then((fileId) => { - opts.finishedCallback(null, fileId); - }) - .catch((err) => { - opts.finishedCallback( - err.message === 'The operation was aborted' ? new Error('Process killed by user') : err, - null, - ); - }); - - return uploadState; + opts.abortSignal, + ); }; download: DownloadStrategyFunction = ( diff --git a/src/lib/core/upload/index.ts b/src/lib/core/upload/index.ts index 1c7d4be..f27462b 100644 --- a/src/lib/core/upload/index.ts +++ b/src/lib/core/upload/index.ts @@ -13,14 +13,13 @@ export type UploadProgressCallback = ( totalBytes: number | null, ) => void; export type EncryptProgressCallback = (progress: number) => void; -export type UploadFinishCallback = (err: Error | null, response: string | null) => void; export interface UploadOptions { progressCallback: UploadProgressCallback; - finishedCallback: UploadFinishCallback; encryptProgressCallback?: EncryptProgressCallback; fileSize: number; source: Readable; + abortSignal?: AbortSignal; } type FileId = string; diff --git a/src/lib/core/upload/multipart.ts b/src/lib/core/upload/multipart.ts index cf3a1ca..0491269 100644 --- a/src/lib/core/upload/multipart.ts +++ b/src/lib/core/upload/multipart.ts @@ -31,7 +31,7 @@ interface PartUpload { source: { size: number; stream: Buffer; index: number }; } -export async function uploadParts(partUrls: string[], stream: Readable, signal: AbortSignal): Promise { +export async function uploadParts(partUrls: string[], stream: Readable, signal?: AbortSignal): Promise { const parts: Part[] = []; const concurrency = 10; diff --git a/src/lib/core/upload/strategy.ts b/src/lib/core/upload/strategy.ts index bdddd06..89a3b5b 100644 --- a/src/lib/core/upload/strategy.ts +++ b/src/lib/core/upload/strategy.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'events'; import { UploadOneStreamStrategyObject, UploadOneShardStrategyObject, UploadOptions } from '.'; -import { Abortable, ActionState, ContractMeta } from '../../../api'; +import { Abortable, ContractMeta } from '../../../api'; import { ShardMeta } from '../../models'; export type NegotiateContract = (shardMeta: ShardMeta) => Promise; @@ -14,7 +14,7 @@ export type UploadStrategyLabel = string; export type UploadStrategyObject = UploadOneStreamStrategyObject | UploadOneShardStrategyObject; -export type UploadStrategyFunction = (bucketId: string, opts: UploadOptions) => ActionState; +export type UploadStrategyFunction = (bucketId: string, opts: UploadOptions) => Promise; export abstract class UploadStrategy extends EventEmitter implements Abortable { fileEncryptionKey: Buffer = Buffer.alloc(0); diff --git a/src/lib/core/upload/uploadV2.ts b/src/lib/core/upload/uploadV2.ts index 3d1eb0e..29f0bde 100644 --- a/src/lib/core/upload/uploadV2.ts +++ b/src/lib/core/upload/uploadV2.ts @@ -5,19 +5,30 @@ import { pipeline as undiciPipeline } from 'undici'; import { validateMnemonic } from 'bip39'; import { uploadFile, uploadMultipartFile } from '@internxt/sdk/dist/network/upload'; -import { ALGORITHMS, Network } from '@internxt/sdk/dist/network'; +import { ALGORITHMS, Crypto, Network } from '@internxt/sdk/dist/network'; import { GenerateFileKey, sha256 } from '../../utils/crypto'; import { Events as ProgressEvents, HashStream, ProgressNotifier } from '../../utils/streams'; -import { ActionState } from '../../../api'; -import { Events } from '..'; import Errors from '../download/errors'; import { UploadProgressCallback } from '.'; import { logger } from '../../utils/logger'; import { uploadParts } from './multipart'; import { AppDetails } from '@internxt/sdk/dist/shared'; -function putStream(url: string, fileSize?: number): Writable { +const MULTIPART_THRESHOLD = 100 * 1024 * 1024; // 100MB + +const crypto: Crypto = { + validateMnemonic: (mnemonic: string) => { + return validateMnemonic(mnemonic); + }, + algorithm: ALGORITHMS.AES256CTR, + generateFileKey: (mnemonic, bucketId, index) => { + return GenerateFileKey(mnemonic, bucketId, index as Buffer); + }, + randomBytes, +}; + +function putStream(url: string, fileSize: number): Writable { const formattedUrl = new URL(url); let headers: Record = { 'Content-Type': 'application/octet-stream', @@ -31,56 +42,19 @@ function putStream(url: string, fileSize?: number): Writable { } export function uploadFileV2( + network: Network, fileSize: number, source: Readable, bucketId: string, mnemonic: string, - bridgeUrl: string, - creds: { pass: string; user: string }, - appDetails: AppDetails, - notifyProgress: UploadProgressCallback, - actionState: ActionState, + progress: ProgressNotifier, + abortSignal?: AbortSignal, ): Promise { - const abortController = new AbortController(); - - actionState.once(Events.Upload.Abort, () => { - abortController.abort(); - }); - - const network = Network.client( - bridgeUrl, - { - ...appDetails, - customHeaders: { - lib: 'inxt-js', - ...appDetails.customHeaders, - }, - }, - { - bridgeUser: creds.user, - userId: sha256(Buffer.from(creds.pass)).toString('hex'), - }, - ); - let cipher: Cipheriv; - const progress = new ProgressNotifier(fileSize, 2000, { emitClose: false }); - - progress.on(ProgressEvents.Progress, (progress: number) => { - notifyProgress(progress, null, null); - }); return uploadFile( network, - { - validateMnemonic: (mnemonic: string) => { - return validateMnemonic(mnemonic); - }, - algorithm: ALGORITHMS.AES256CTR, - generateFileKey: (mnemonic, bucketId, index) => { - return GenerateFileKey(mnemonic, bucketId, index as Buffer); - }, - randomBytes, - }, + crypto, bucketId, mnemonic, fileSize, @@ -93,13 +67,13 @@ export function uploadFileV2( cipher = createCipheriv('aes-256-ctr', key as Buffer, iv as Buffer); }, - async (url: string) => { + async (url) => { logger.debug('Uploading file to %s...', url); const hasher = new HashStream(); await pipeline(source, cipher, hasher, progress, putStream(url, fileSize), { - signal: abortController.signal, + signal: abortSignal, }); const fileHash = hasher.getHash().toString('hex'); @@ -108,62 +82,26 @@ export function uploadFileV2( return fileHash; }, + abortSignal, ); } export function uploadFileMultipart( + network: Network, fileSize: number, source: Readable, bucketId: string, mnemonic: string, - bridgeUrl: string, - creds: { pass: string; user: string }, - notifyProgress: UploadProgressCallback, - actionState: ActionState, - appDetails: AppDetails, + progress: ProgressNotifier, + abortSignal?: AbortSignal, ): Promise { - const abortController = new AbortController(); - - actionState.once(Events.Upload.Abort, () => { - abortController.abort(); - }); - - const network = Network.client( - bridgeUrl, - { - ...appDetails, - customHeaders: { - lib: 'inxt-js', - ...appDetails.customHeaders, - }, - }, - { - bridgeUser: creds.user, - userId: sha256(Buffer.from(creds.pass)).toString('hex'), - }, - ); - let cipher: Cipheriv; - const progress = new ProgressNotifier(fileSize, 2000, { emitClose: false }); const partSize = 15 * 1024 * 1024; const parts = Math.ceil(fileSize / partSize); - progress.on(ProgressEvents.Progress, (progress: number) => { - notifyProgress(progress, null, null); - }); - return uploadMultipartFile( network, - { - validateMnemonic: (mnemonic: string) => { - return validateMnemonic(mnemonic); - }, - algorithm: ALGORITHMS.AES256CTR, - generateFileKey: (mnemonic, bucketId, index) => { - return GenerateFileKey(mnemonic, bucketId, index as Buffer); - }, - randomBytes, - }, + crypto, bucketId, mnemonic, fileSize, @@ -181,17 +119,13 @@ export function uploadFileMultipart( const hasher = new HashStream(); const pipelineToFinish = pipeline(source, cipher, hasher, progress, { - signal: abortController.signal, + signal: abortSignal, }); - const parts = await uploadParts(urls, progress, abortController.signal); + const parts = await uploadParts(urls, progress, abortSignal); await pipelineToFinish; - if (abortController.signal.aborted) { - throw new Error('Process killed by user'); - } - const fileHash = hasher.getHash().toString('hex'); logger.debug('File uploaded (hash %s)', fileHash); @@ -201,6 +135,46 @@ export function uploadFileMultipart( parts, }; }, + abortSignal, parts, ); } + +export function upload( + fileSize: number, + source: Readable, + bucketId: string, + mnemonic: string, + bridgeUrl: string, + creds: { pass: string; user: string }, + appDetails: AppDetails, + notifyProgress: UploadProgressCallback, + abortSignal?: AbortSignal, +): Promise { + const network = Network.client( + bridgeUrl, + { + ...appDetails, + customHeaders: { + lib: 'inxt-js', + ...appDetails.customHeaders, + }, + }, + { + bridgeUser: creds.user, + userId: sha256(Buffer.from(creds.pass)).toString('hex'), + }, + ); + + const progress = new ProgressNotifier(fileSize, 2000, { emitClose: false }); + + progress.on(ProgressEvents.Progress, (progress: number) => { + notifyProgress(progress, null, null); + }); + + if (fileSize > MULTIPART_THRESHOLD) { + return uploadFileMultipart(network, fileSize, source, bucketId, mnemonic, progress, abortSignal); + } + + return uploadFileV2(network, fileSize, source, bucketId, mnemonic, progress, abortSignal); +} diff --git a/tests/lib/core/upload/uploadV2.test.ts b/tests/lib/core/upload/upload.test.ts similarity index 83% rename from tests/lib/core/upload/uploadV2.test.ts rename to tests/lib/core/upload/upload.test.ts index 9dffa26..3c0ad3b 100644 --- a/tests/lib/core/upload/uploadV2.test.ts +++ b/tests/lib/core/upload/upload.test.ts @@ -2,8 +2,7 @@ import { describe, expect, it } from 'vitest'; import { fail } from 'node:assert'; import { Readable } from 'stream'; import { UploadInvalidMnemonicError } from '@internxt/sdk/dist/network/errors'; -import { ActionState, ActionTypes } from '../../../../src/api'; -import { uploadFileV2 } from '../../../../src/lib/core/upload/uploadV2'; +import { upload } from '../../../../src/lib/core/upload/uploadV2'; import { getBridgeUrl, getBucketId, @@ -16,16 +15,17 @@ import { const creds = getNetworkCredentials(); const bucketId = getBucketId(); const bridgeUrl = getBridgeUrl(); +const abortSignal = new AbortController().signal; const fileContent = 'some text that i have in the file'; const fileBytes = getFileBytes(fileContent); const validMnemonic = getValidMnemonic(); const invalidMnemonic = getInvalidMnemonic(); -describe('uploadFileV2()', () => { +describe('upload()', () => { describe('Should handle errors properly', () => { it('Should throw if the mnemonic is invalid', async () => { try { - await uploadFileV2( + await upload( 0, Readable.from(fileBytes), bucketId, @@ -34,7 +34,7 @@ describe('uploadFileV2()', () => { creds, { clientName: 'inxt-js', clientVersion: '1.0' }, () => {}, - new ActionState(ActionTypes.Upload), + abortSignal, ); fail('Expected function to throw an error, but it did not.'); diff --git a/yarn.lock b/yarn.lock index 4b7eef1..97656b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -412,10 +412,10 @@ version "1.0.2" resolved "https://codeload.github.com/internxt/prettier-config/tar.gz/9fa74e9a2805e1538b50c3809324f1c9d0f3e4f9" -"@internxt/sdk@1.15.1": - version "1.15.1" - resolved "https://registry.yarnpkg.com/@internxt/sdk/-/sdk-1.15.1.tgz#69ad13a3c8cacbd929f025f24adc6fd2d1faf8ef" - integrity sha512-CEH/fNjDWenmFAl8NHaykb85AY4uEZDO5pA2ap8/GD4SXmxC3rcWsKnMsYO9jc8R2vpOPxmE3oBlNqOVcI/llQ== +"@internxt/sdk@1.15.2": + version "1.15.2" + resolved "https://registry.yarnpkg.com/@internxt/sdk/-/sdk-1.15.2.tgz#240dfb0f9ad3d18bf0ea32a36b15aabc5cc6bed9" + integrity sha512-Hial7LUBGQ0nUTQc9G6ANRCYyVseGA73JG9ZMQCywkqGlEBslrLod0TuxyvrEcvFLkfS4iNvATWGZGNFMsMceg== dependencies: axios "1.13.5" internxt-crypto "0.0.13"