Skip to content

feat: refactor sync requests to use driveServerClient #267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
egalvis27 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: refactor sync requests to use driveServerClient #267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@
"async": "^3.2.4",
"axios": "^1.1.3",
"better-sqlite3": "^11.10.0",
"bottleneck": "^2.19.5",
"check-disk-space": "^3.4.0",
"diod": "^2.0.0",
"electron-updater": "^4.6.4",
Expand Down
45 changes: 22 additions & 23 deletions src/apps/main/remote-sync/RemoteSyncManager.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
vi.mock('@internxt/drive-desktop-core/build/backend');
vi.mock('axios');
vi.mock('../../../infra/drive-server/client/drive-server.client.instance', () => ({
driveServerClient: {
GET: vi.fn(),
},
}));
vi.mock('./RemoteSyncErrorHandler/RemoteSyncErrorHandler', () => ({
RemoteSyncErrorHandler: vi.fn().mockImplementation(() => ({
handleSyncError: vi.fn(),
Expand All @@ -15,14 +19,15 @@ import { RemoteSyncErrorHandler } from './RemoteSyncErrorHandler/RemoteSyncError
import { RemoteSyncManager } from './RemoteSyncManager';
import { RemoteSyncedFile, RemoteSyncedFolder } from './helpers';
import * as uuid from 'uuid';
import axios from 'axios';
import { DriveServerError } from '../../../infra/drive-server/drive-server.error';
import { driveServerClient } from '../../../infra/drive-server/client/drive-server.client.instance';
import { DatabaseCollectionAdapter } from '../database/adapters/base';
import { DriveFile } from '../database/entities/DriveFile';
import { DriveFolder } from '../database/entities/DriveFolder';
import { createOrUpdateFileByBatch } from '../../../infra/sqlite/services/file/create-or-update-file-by-batch';
import { createOrUpdateFolderByBatch } from '../../../infra/sqlite/services/folder/create-or-update-folder-by-batch';

const mockedAxios = vi.mocked(axios);
const mockedGet = vi.mocked(driveServerClient.GET);
const mockedCreateOrUpdateFileByBatch = vi.mocked(createOrUpdateFileByBatch);
const mockedCreateOrUpdateFolderByBatch = vi.mocked(createOrUpdateFolderByBatch);

Expand Down Expand Up @@ -101,15 +106,14 @@ describe('RemoteSyncManager', () => {
files: inMemorySyncedFilesCollection,
},
{
httpClient: mockedAxios,
fetchFilesLimitPerRequest: 2,
fetchFoldersLimitPerRequest: 2,
syncFiles: true,
syncFolders: true,
},
errorHandler,
);
mockedAxios.get.mockClear();
mockedGet.mockClear();
mockedCreateOrUpdateFileByBatch.mockClear();
mockedCreateOrUpdateFolderByBatch.mockClear();
});
Expand All @@ -122,7 +126,6 @@ describe('RemoteSyncManager', () => {
files: inMemorySyncedFilesCollection,
},
{
httpClient: mockedAxios,
fetchFilesLimitPerRequest: 2,
fetchFoldersLimitPerRequest: 2,
syncFiles: true,
Expand All @@ -131,7 +134,7 @@ describe('RemoteSyncManager', () => {
errorHandler,
);

mockedAxios.get
mockedGet
.mockResolvedValueOnce({
data: [
createRemoteSyncedFileFixture({
Expand All @@ -152,7 +155,7 @@ describe('RemoteSyncManager', () => {

await sut.startRemoteSync();

expect(mockedAxios.get).toBeCalledTimes(2);
expect(mockedGet).toBeCalledTimes(2);
expect(sut.getSyncStatus()).toBe('SYNCED');
});
it('Should sync all the folders', async () => {
Expand All @@ -162,7 +165,6 @@ describe('RemoteSyncManager', () => {
files: inMemorySyncedFilesCollection,
},
{
httpClient: mockedAxios,
fetchFilesLimitPerRequest: 2,
fetchFoldersLimitPerRequest: 2,
syncFiles: false,
Expand All @@ -171,7 +173,7 @@ describe('RemoteSyncManager', () => {
errorHandler,
);

mockedAxios.get
mockedGet
.mockResolvedValueOnce({
data: [
createRemoteSyncedFolderFixture({
Expand Down Expand Up @@ -202,7 +204,7 @@ describe('RemoteSyncManager', () => {

await sut.startRemoteSync();

expect(mockedAxios.get).toBeCalledTimes(3);
expect(mockedGet).toBeCalledTimes(3);
expect(sut.getSyncStatus()).toBe('SYNCED');
});

Expand All @@ -213,7 +215,6 @@ describe('RemoteSyncManager', () => {
files: inMemorySyncedFilesCollection,
},
{
httpClient: mockedAxios,
fetchFilesLimitPerRequest: 2,
fetchFoldersLimitPerRequest: 2,
syncFiles: true,
Expand All @@ -229,34 +230,34 @@ describe('RemoteSyncManager', () => {
plainName: 'file_2',
});

mockedAxios.get.mockResolvedValueOnce({ data: [file1, file2] });
mockedGet.mockResolvedValueOnce({ data: [file1, file2] });

mockedAxios.get.mockResolvedValueOnce({ data: [] });
mockedGet.mockResolvedValueOnce({ data: [] });

await sut.startRemoteSync();

expect(mockedAxios.get).toBeCalledTimes(2);
expect(mockedGet).toBeCalledTimes(2);
expect(sut.getSyncStatus()).toBe('SYNCED');
expect(mockedCreateOrUpdateFileByBatch).toBeCalledWith({ files: [file1, file2] });
});
});

describe('When something fails during the sync', () => {
it('Should retry N times and then stop if sync does not succeed', async () => {
mockedAxios.get.mockImplementation(() => Promise.reject('Fail on purpose'));
mockedGet.mockResolvedValue({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') });

await sut.startRemoteSync();

expect(mockedAxios.get).toBeCalledTimes(6);
expect(mockedGet).toBeCalledTimes(6);
expect(sut.getSyncStatus()).toBe('SYNC_FAILED');
});

it('Should fail the sync if some files or folders cannot be retrieved', async () => {
mockedAxios.get.mockRejectedValueOnce('Fail on purpose');
mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') });

await sut.startRemoteSync();

expect(mockedAxios.get).toBeCalledTimes(6);
expect(mockedGet).toBeCalledTimes(6);
expect(sut.getSyncStatus()).toBe('SYNC_FAILED');
});

Expand All @@ -267,15 +268,14 @@ describe('RemoteSyncManager', () => {
files: inMemorySyncedFilesCollection,
},
{
httpClient: mockedAxios,
fetchFilesLimitPerRequest: 2,
fetchFoldersLimitPerRequest: 2,
syncFiles: true,
syncFolders: false,
},
errorHandler,
);
mockedAxios.get.mockRejectedValueOnce('Fail on purpose');
mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') });
const errorHandlerInstance = sut['errorHandler'];
const errorHandlerSpy = vi.spyOn(errorHandlerInstance, 'handleSyncError');

Expand All @@ -292,7 +292,6 @@ describe('RemoteSyncManager', () => {
files: inMemorySyncedFilesCollection,
},
{
httpClient: mockedAxios,
fetchFilesLimitPerRequest: 2,
fetchFoldersLimitPerRequest: 2,
syncFiles: false,
Expand All @@ -301,7 +300,7 @@ describe('RemoteSyncManager', () => {
errorHandler,
);

mockedAxios.get.mockRejectedValueOnce('Fail on purpose');
mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') });
const errorHandlerInstance = sut['errorHandler'];
const errorHandlerSpy = vi.spyOn(errorHandlerInstance, 'handleSyncError');

Expand Down
129 changes: 48 additions & 81 deletions src/apps/main/remote-sync/RemoteSyncManager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,14 @@ import {
SIX_HOURS_IN_MILLISECONDS,
} from './helpers';
import { DatabaseCollectionAdapter } from '../database/adapters/base';
import axios, { Axios } from 'axios';
import { DriveFolder } from '../database/entities/DriveFolder';
import { DriveFile } from '../database/entities/DriveFile';
import { Nullable } from '../../shared/types/Nullable';
import {
RemoteSyncError,
RemoteSyncInvalidResponseError,
RemoteSyncNetworkError,
RemoteSyncServerError,
} from './errors';
import { RemoteSyncError, RemoteSyncInvalidResponseError, RemoteSyncNetworkError } from './errors';
import { RemoteSyncErrorHandler } from './RemoteSyncErrorHandler/RemoteSyncErrorHandler';
import { createOrUpdateFolderByBatch } from '../../../infra/sqlite/services/folder/create-or-update-folder-by-batch';
import { createOrUpdateFileByBatch } from '../../../infra/sqlite/services/file/create-or-update-file-by-batch';
import { driveServerClient } from '../../../infra/drive-server/client/drive-server.client.instance';

export class RemoteSyncManager {
private foldersSyncStatus: RemoteSyncStatus = 'IDLE';
Expand All @@ -36,7 +31,6 @@ export class RemoteSyncManager {
folders: DatabaseCollectionAdapter<DriveFolder>;
},
private config: {
httpClient: Axios;
fetchFilesLimitPerRequest: number;
fetchFoldersLimitPerRequest: number;
syncFiles: boolean;
Expand Down Expand Up @@ -335,47 +329,33 @@ export class RemoteSyncManager {
hasMore: boolean;
result: RemoteSyncedFile[];
}> {
const params = {
limit: this.config.fetchFilesLimitPerRequest,
offset: 0,
status: 'ALL',
updatedAt: updatedAtCheckpoint ? updatedAtCheckpoint.toISOString() : undefined,
};

try {
const response = await this.config.httpClient.get(`${process.env.NEW_DRIVE_URL}/files`, {
params,
});

if (response.status > 299) {
throw new RemoteSyncServerError(response.status, response.data);
}

if (!Array.isArray(response.data)) {
logger.debug({
tag: 'SYNC-ENGINE',
msg: `Expected to receive an array of files, but received: ${JSON.stringify(response, null, 2)}`,
});
throw new RemoteSyncInvalidResponseError(response);
}
const { data, error } = await driveServerClient.GET('/files', {
query: {
limit: this.config.fetchFilesLimitPerRequest,
offset: 0,
status: 'ALL',
updatedAt: updatedAtCheckpoint?.toISOString(),
},
});
Comment on lines +332 to +339
Copy link

@AlexisMora AlexisMora Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make this a reusable function inside src/infra/drive-server/files/services where we store the rest of the file related api calls.
This way its reusable. we can test it in isolation and we also hide complexity to where its needed.


const hasMore = response.data.length === this.config.fetchFilesLimitPerRequest;
if (error) {
throw new RemoteSyncNetworkError(error.message, undefined, error.statusCode);
}

return {
hasMore,
result: response.data && Array.isArray(response.data) ? response.data.map(this.patchDriveFileResponseItem) : [],
};
} catch (error) {
if (error instanceof RemoteSyncError) {
throw error;
}
if (!Array.isArray(data)) {
logger.debug({
tag: 'SYNC-ENGINE',
msg: `Expected to receive an array of files, but received: ${JSON.stringify(data, null, 2)}`,
});
throw new RemoteSyncInvalidResponseError(data);
}

if (axios.isAxiosError(error)) {
throw new RemoteSyncNetworkError(error.message, error.code, error.response?.status);
}
const hasMore = data.length === this.config.fetchFilesLimitPerRequest;

throw new RemoteSyncError('Uncontrolled Error in fetchFilesFromRemote', undefined, { originalError: error });
}
return {
hasMore,
result: data.map(this.patchDriveFileResponseItem),
};
}

/**
Expand All @@ -387,46 +367,33 @@ export class RemoteSyncManager {
hasMore: boolean;
result: RemoteSyncedFolder[];
}> {
const params = {
limit: this.config.fetchFilesLimitPerRequest,
offset: 0,
status: 'ALL',
updatedAt: updatedAtCheckpoint ? updatedAtCheckpoint.toISOString() : undefined,
};
try {
const response = await this.config.httpClient.get(`${process.env.NEW_DRIVE_URL}/folders`, {
params,
});
if (response.status > 299) {
throw new RemoteSyncServerError(response.status, response.data);
}

if (!Array.isArray(response.data)) {
logger.debug({
tag: 'SYNC-ENGINE',
msg: `Expected to receive an array of folders, but instead received: ${JSON.stringify(response, null, 2)}`,
});
throw new RemoteSyncInvalidResponseError(response);
}
const { data, error } = await driveServerClient.GET('/folders', {
query: {
limit: this.config.fetchFilesLimitPerRequest,
offset: 0,
status: 'ALL',
updatedAt: updatedAtCheckpoint?.toISOString(),
},
});
Comment on lines +370 to +377
Copy link

@AlexisMora AlexisMora Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The same


const hasMore = response.data.length === this.config.fetchFilesLimitPerRequest;
if (error) {
throw new RemoteSyncNetworkError(error.message, undefined, error.statusCode);
}

return {
hasMore,
result:
response.data && Array.isArray(response.data) ? response.data.map(this.patchDriveFolderResponseItem) : [],
};
} catch (error) {
if (error instanceof RemoteSyncError) {
throw error;
}
if (!Array.isArray(data)) {
logger.debug({
tag: 'SYNC-ENGINE',
msg: `Expected to receive an array of folders, but instead received: ${JSON.stringify(data, null, 2)}`,
});
throw new RemoteSyncInvalidResponseError(data);
}

if (axios.isAxiosError(error)) {
throw new RemoteSyncNetworkError(error.message, error.code, error.response?.status);
}
const hasMore = data.length === this.config.fetchFilesLimitPerRequest;

throw new RemoteSyncError('Uncontrolled Error in fetchFoldersFromRemote', undefined, { originalError: error });
}
return {
hasMore,
result: data.map(this.patchDriveFolderResponseItem),
};
}

private patchDriveFolderResponseItem = (payload: any): RemoteSyncedFolder => {
Expand Down
2 changes: 0 additions & 2 deletions src/apps/main/remote-sync/service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { debounce } from 'lodash';
import eventBus from '../event-bus';
import { getNewTokenClient } from '../../shared/HttpClient/main-process-client';
import { DriveFilesCollection } from '../database/collections/DriveFileCollection';
import { DriveFoldersCollection } from '../database/collections/DriveFolderCollection';
import { RemoteSyncManager } from './RemoteSyncManager';
Expand All @@ -20,7 +19,6 @@ export const remoteSyncManager = new RemoteSyncManager(
folders: driveFoldersCollection,
},
{
httpClient: getNewTokenClient(),
fetchFilesLimitPerRequest: 1000,
fetchFoldersLimitPerRequest: 1000,
syncFiles: true,
Expand Down
Loading
Loading