From 84806b9a8691a0ae934bbfdfce058b4131e1be12 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 5 Nov 2025 10:55:12 -0300 Subject: [PATCH 01/15] added build:watch to help development --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index e9575ad..e6265bf 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ ], "scripts": { "build": "npm run build --workspaces", + "build:watch": "npm run build:watch --workspaces", "format": "npm run format --workspaces", "lint": "npm run lint --workspaces", "prepare": "husky install || true", From d379cae0c03fdea4b3eb9b22b914fc7a44072bb6 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 5 Nov 2025 10:55:58 -0300 Subject: [PATCH 02/15] refactored assignment creation to avoid delay with firebase background functions --- .../levante-admin/src/upsertAdministration.ts | 229 +++++++++++++++++- 1 file changed, 224 insertions(+), 5 deletions(-) diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index 8db727f..50f0952 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -6,8 +6,25 @@ import { } from "firebase-admin/firestore"; import { HttpsError } from "firebase-functions/v2/https"; import { logger } from "firebase-functions/v2"; -import type { IAssessment, IOrgsList } from "./interfaces.js"; // Assuming necessary types/helpers are in common import type { Class, Group, School } from "../firestore-schema.js"; +import type { IAssessment, IOrgsList, IAdministration } from "./interfaces.js"; // Assuming necessary types/helpers are in common +import { ORG_NAMES } from "./interfaces.js"; +import { standardizeAdministrationOrgs } from "./administrations/administration-utils.js"; +import { updateAssignmentsForOrgChunkHandler } from "./assignments/sync-assignments.js"; +import _reduce from "lodash-es/reduce.js"; +import _pick from "lodash-es/pick.js"; +import _difference from "lodash-es/difference.js"; +import _fromPairs from "lodash-es/fromPairs.js"; +import _map from "lodash-es/map.js"; +import { + chunkOrgs, + getOnlyExistingOrgs, + getExhaustiveOrgs, + getUsersFromOrgs, +} from "./orgs/org-utils.js"; +import { removeOrgsFromAssignments } from "./assignments/assignment-utils.js"; +import _chunk from "lodash-es/chunk.js"; +import { MAX_TRANSACTIONS } from "./utils/utils.js"; interface UpsertAdministrationData { name: string; @@ -50,6 +67,149 @@ interface IAdministrationDoc { creatorName: string; } +const syncNewAdministrationAssignments = async ( + administrationId: string, + administrationDocRef: DocumentReference, + currData: IAdministration +) => { + const { minimalOrgs } = await standardizeAdministrationOrgs({ + administrationId, + administrationDocRef, + currData, + copyToSubCollections: true, + forceCopy: true, + }); + + const orgChunks = chunkOrgs(minimalOrgs, 100); + const maxChunksToProcessSync = 3; + + for (let i = 0; i < Math.min(orgChunks.length, maxChunksToProcessSync); i++) { + const orgChunk = orgChunks[i]; + await updateAssignmentsForOrgChunkHandler({ + administrationId, + administrationData: currData, + orgChunk, + mode: "add", + }); + } + + if (orgChunks.length > maxChunksToProcessSync) { + logger.info( + `Processed first ${maxChunksToProcessSync} org chunks synchronously. Remaining chunks will be processed by background trigger.`, + { + administrationId, + totalChunks: orgChunks.length, + processedChunks: maxChunksToProcessSync, + } + ); + } +}; + +const syncModifiedAdministrationAssignments = async ( + administrationId: string, + administrationDocRef: DocumentReference, + prevData: IAdministration, + currData: IAdministration +) => { + const db = getFirestore(); + const prevOrgs: IOrgsList = _pick(prevData, ORG_NAMES); + const currOrgs: IOrgsList = _pick(currData, ORG_NAMES); + + const removedOrgs = _fromPairs( + _map(Object.entries(currOrgs), ([key, value]) => [ + key, + _difference(prevOrgs[key], value), + ]) + ) as IOrgsList; + + const numRemovedOrgs = _reduce( + removedOrgs, + (sum, value) => (value ? sum + value.length : sum), + 0 + ); + + if (numRemovedOrgs > 0) { + let remainingUsersToRemove: string[] = []; + let removedExhaustiveOrgs: IOrgsList = {}; + + await db.runTransaction(async (transaction) => { + const removedExistingOrgs = await getOnlyExistingOrgs( + removedOrgs, + transaction + ); + removedExhaustiveOrgs = await getExhaustiveOrgs({ + orgs: removedExistingOrgs, + transaction, + includeArchived: true, + }); + const usersToRemove = await getUsersFromOrgs({ + orgs: removedExhaustiveOrgs, + transaction, + includeArchived: true, + }); + + if (usersToRemove.length !== 0) { + if (usersToRemove.length <= MAX_TRANSACTIONS) { + return removeOrgsFromAssignments( + usersToRemove, + [administrationId], + removedExhaustiveOrgs, + transaction + ); + } else { + remainingUsersToRemove = usersToRemove; + return Promise.resolve(usersToRemove.length); + } + } else { + return Promise.resolve(0); + } + }); + + for (const _userChunk of _chunk(remainingUsersToRemove, MAX_TRANSACTIONS)) { + await db.runTransaction(async (transaction) => { + return removeOrgsFromAssignments( + _userChunk, + [administrationId], + removedExhaustiveOrgs, + transaction + ); + }); + } + } + + const { minimalOrgs } = await standardizeAdministrationOrgs({ + administrationId, + administrationDocRef, + currData, + copyToSubCollections: true, + forceCopy: true, + }); + + const orgChunks = chunkOrgs(minimalOrgs, 100); + const maxChunksToProcessSync = 3; + + for (let i = 0; i < Math.min(orgChunks.length, maxChunksToProcessSync); i++) { + const orgChunk = orgChunks[i]; + await updateAssignmentsForOrgChunkHandler({ + administrationId, + administrationData: currData, + orgChunk, + mode: "update", + }); + } + + if (orgChunks.length > maxChunksToProcessSync) { + logger.info( + `Processed first ${maxChunksToProcessSync} org chunks synchronously. Remaining chunks will be processed by background trigger.`, + { + administrationId, + totalChunks: orgChunks.length, + processedChunks: maxChunksToProcessSync, + } + ); + } +}; + export const upsertAdministrationHandler = async ( callerAdminUid: string, data: UpsertAdministrationData @@ -180,12 +340,22 @@ export const upsertAdministrationHandler = async ( // 5. Firestore Transaction try { + let prevData: IAdministration | undefined; + + if (administrationId) { + const prevDoc = await db + .collection("administrations") + .doc(administrationId) + .get(); + if (prevDoc.exists) { + prevData = prevDoc.data() as IAdministration; + } + } + const newAdministrationId = await db.runTransaction(async (transaction) => { let administrationDocRef: DocumentReference; - let operationType: string; // To log 'create' or 'update' if (administrationId) { - operationType = "update"; administrationDocRef = db .collection("administrations") .doc(administrationId); @@ -239,7 +409,6 @@ export const upsertAdministrationHandler = async ( transaction.update(administrationDocRef, updateData); // Switched from set with merge to update } else { // --- Create Path --- - operationType = "create"; administrationDocRef = db.collection("administrations").doc(); // --- Read 1 (Create Path) --- Check if user doc exists BEFORE any writes @@ -363,8 +532,9 @@ export const upsertAdministrationHandler = async ( ); } } - logger.info(`Successfully prepared administration ${operationType}`, { + logger.info("Successfully prepared administration", { administrationId: administrationDocRef.id, + operationType: administrationId ? "update" : "create", }); return administrationDocRef.id; // Return the ID }); // End Transaction @@ -372,6 +542,55 @@ export const upsertAdministrationHandler = async ( logger.info("Finished administration upsert transaction", { administrationId: newAdministrationId, }); + + // Sync assignments synchronously for immediate visibility + try { + const administrationDocRef = db + .collection("administrations") + .doc(newAdministrationId); + const administrationDoc = await administrationDocRef.get(); + + if (!administrationDoc.exists) { + logger.warn( + `Administration ${newAdministrationId} not found after creation. Skipping sync.` + ); + return { status: "ok", administrationId: newAdministrationId }; + } + + const administrationData = administrationDoc.data() as IAdministration; + const isNewAdministration = !administrationId; + + if (isNewAdministration) { + await syncNewAdministrationAssignments( + newAdministrationId, + administrationDocRef, + administrationData + ); + } else if (prevData) { + await syncModifiedAdministrationAssignments( + newAdministrationId, + administrationDocRef, + prevData, + administrationData + ); + } else { + await syncNewAdministrationAssignments( + newAdministrationId, + administrationDocRef, + administrationData + ); + } + + logger.info("Finished synchronous assignment sync", { + administrationId: newAdministrationId, + }); + } catch (syncError: any) { + logger.error("Error during synchronous assignment sync", { + error: syncError, + administrationId: newAdministrationId, + }); + } + return { status: "ok", administrationId: newAdministrationId }; } catch (error: any) { logger.error("Error during administration upsert", { error }); From 5626283973771a9d1b3a0676d2287045e5c46742 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 5 Nov 2025 10:56:11 -0300 Subject: [PATCH 03/15] added README to explain the changes with assignment creation --- .../SYNCHRONOUS_ASSIGNMENT_SYNC.md | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md diff --git a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md new file mode 100644 index 0000000..ccd21b8 --- /dev/null +++ b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md @@ -0,0 +1,193 @@ +# Synchronous Assignment Sync + +## Overview + +This document explains the changes made to eliminate the delay between creating/editing an assignment and displaying it on the home page. Previously, assignment creation/updates triggered background Firebase functions that processed assignments asynchronously, causing users to not see their newly created or edited assignments immediately. + +## Problem + +When a user created or edited an assignment via `upsertAdministration`: + +1. The administration document was created/updated in Firestore +2. A Firestore trigger (`syncAssignmentsOnAdministrationUpdate`) was fired asynchronously +3. This trigger processed assignments in the background using task queues +4. Users were redirected to the home page before assignments were visible +5. This created a poor user experience with a noticeable delay + +## Solution + +The solution implements **synchronous assignment processing** within the `upsertAdministration` handler, ensuring that critical assignment operations complete before the function returns. This provides immediate visibility of assignments while maintaining the background trigger for eventual consistency. + +## Implementation Details + +### Architecture + +The implementation follows a two-tier approach: + +1. **Synchronous Processing (Primary)**: Processes the first 3 org chunks (up to ~300 organizations) immediately +2. **Background Processing (Secondary)**: The existing Firestore trigger continues to handle: + - Remaining org chunks for large assignments + - Eventual consistency + - Edge cases and retries + +### Code Structure + +The synchronous sync logic is implemented as two helper functions within `upsertAdministration.ts`: + +- `syncNewAdministrationAssignments` - Handles new administration creation +- `syncModifiedAdministrationAssignments` - Handles administration updates + +These functions are called directly after the Firestore transaction completes, ensuring assignments are processed before the function returns to the client. + +### Key Functions + +#### `syncNewAdministrationAssignments` + +Handles synchronous assignment creation for new administrations: + +```typescript +const syncNewAdministrationAssignments = async ( + administrationId: string, + administrationDocRef: DocumentReference, + currData: IAdministration +) +``` + +**Process:** +1. Standardizes administration orgs using `standardizeAdministrationOrgs` +2. Chunks orgs into groups of 100 +3. Processes the first 3 chunks synchronously using `updateAssignmentsForOrgChunkHandler` +4. Logs if additional chunks will be processed by the background trigger + +#### `syncModifiedAdministrationAssignments` + +Handles synchronous assignment updates for modified administrations: + +```typescript +const syncModifiedAdministrationAssignments = async ( + administrationId: string, + administrationDocRef: DocumentReference, + prevData: IAdministration, + currData: IAdministration +) +``` + +**Process:** +1. Calculates removed orgs by comparing previous and current org lists +2. Removes assignments from users in removed orgs +3. Standardizes administration orgs +4. Processes the first 3 org chunks synchronously for updates + +### Modified Flow + +**Before:** +``` +upsertAdministration → Transaction → Return + ↓ + Firestore Trigger (async) → Process Assignments +``` + +**After:** +``` +upsertAdministration → Transaction → Sync Assignments (sync) → Return + ↓ + Firestore Trigger (async) → Process Remaining Chunks +``` + +### Implementation Details + +The synchronous sync happens immediately after the transaction commits: + +1. The administration document is fetched from Firestore (to ensure server timestamps are populated) +2. Based on whether it's a create or update operation, the appropriate sync function is called +3. The sync function processes org chunks synchronously +4. The function returns, allowing the client to see the assignment immediately +5. The background trigger continues processing any remaining chunks asynchronously + +### Integration with Background Trigger + +The background Firestore trigger (`syncAssignmentsOnAdministrationUpdate`) continues to operate: + +- **Idempotent Operations**: All assignment operations are idempotent, so duplicate processing is safe +- **Large Scale**: For assignments with more than 3 org chunks, the background trigger processes remaining chunks +- **Resilience**: Provides backup processing if synchronous processing encounters errors +- **Eventual Consistency**: Ensures all assignments are eventually processed correctly + +## Configuration + +### Synchronous Processing Limits + +The implementation processes the first **3 org chunks** synchronously, where each chunk contains up to **100 organizations**. This balances: + +- **Immediate Visibility**: Users see assignments immediately for most common scenarios +- **Function Timeout**: Prevents Cloud Functions from timing out on very large assignments +- **Performance**: Keeps response times reasonable + +These limits can be adjusted by modifying the `maxChunksToProcessSync` constant in: +- `syncNewAdministrationAssignments` +- `syncModifiedAdministrationAssignments` + +## Error Handling + +The synchronous sync is wrapped in a try-catch block that: + +- Logs errors without failing the main `upsertAdministration` operation +- Allows the function to return successfully even if sync fails +- Relies on the background trigger as a fallback for error recovery + +This ensures that assignment creation/updates are not blocked by sync errors. Even if the synchronous sync encounters an error, the administration document is still created/updated successfully, and the background trigger will process the assignments asynchronously. + +### Error Recovery + +If synchronous sync fails: +1. The error is logged with full context +2. The `upsertAdministration` function still returns successfully +3. The background Firestore trigger will process all assignments +4. Users may experience a slight delay, but assignments will appear eventually + +## Performance Considerations + +### Function Execution Time + +- **Small Assignments** (≤3 chunks): Fully processed synchronously +- **Large Assignments** (>3 chunks): First 3 chunks processed synchronously, remainder processed in background + +### Transaction Limits + +The implementation respects Firestore transaction limits: +- Maximum of `MAX_TRANSACTIONS` (100) documents per transaction +- Large user sets are processed in chunks across multiple transactions + +## Testing + +When testing this functionality: + +1. **Verify Immediate Visibility**: Create an assignment and confirm it appears immediately on the home page +2. **Test Large Assignments**: Create assignments with many orgs to verify background processing continues +3. **Test Updates**: Edit existing assignments to verify synchronous updates work correctly +4. **Monitor Logs**: Check for synchronous sync completion logs and background trigger execution + +## Future Improvements + +Potential enhancements: + +1. **Configurable Chunk Limits**: Make `maxChunksToProcessSync` configurable via environment variables +2. **Progress Tracking**: Add metadata to track which chunks have been processed synchronously +3. **Skip Background Processing**: Optionally skip background trigger for fully-synced assignments +4. **Metrics**: Track synchronous vs. background processing ratios + +## Related Files + +- `functions/levante-admin/src/upsertAdministration.ts` - Main handler with synchronous sync +- `functions/levante-admin/src/assignments/sync-assignments.ts` - Assignment sync utilities +- `functions/levante-admin/src/administrations/sync-administrations.ts` - Background sync handlers +- `functions/levante-admin/src/index.ts` - Firestore trigger definitions + +## Notes + +- The background trigger (`syncAssignmentsOnAdministrationUpdate`) will still fire after document writes +- Duplicate processing is safe due to idempotent operations +- The synchronous processing uses the same underlying functions as the background trigger for consistency +- The helper functions (`syncNewAdministrationAssignments` and `syncModifiedAdministrationAssignments`) are defined as private functions within the `upsertAdministration.ts` file +- For update operations, the previous administration data is fetched before the transaction to enable proper comparison of org changes + From b39df0106e833a8a268004925a556a70990f9774 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Mon, 10 Nov 2025 10:42:36 -0300 Subject: [PATCH 04/15] added rollback function to revert assignment creation if not all the participants receive the assignment --- .../src/assignments/assignment-utils.ts | 120 +++++++++++++- .../src/assignments/on-assignment-updates.ts | 54 +++++++ .../src/assignments/sync-assignments.ts | 150 ++++++++++++------ .../levante-admin/src/upsertAdministration.ts | 100 ++++++++++-- 4 files changed, 356 insertions(+), 68 deletions(-) diff --git a/functions/levante-admin/src/assignments/assignment-utils.ts b/functions/levante-admin/src/assignments/assignment-utils.ts index bfb444b..3f75c1b 100644 --- a/functions/levante-admin/src/assignments/assignment-utils.ts +++ b/functions/levante-admin/src/assignments/assignment-utils.ts @@ -1,5 +1,10 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { getFirestore, Timestamp } from "firebase-admin/firestore"; +import { + getFirestore, + Timestamp, + FieldValue, + FieldPath, +} from "firebase-admin/firestore"; import type { CollectionReference, DocumentData, @@ -115,6 +120,96 @@ export const removeAssignmentFromUsers = async ( ); }; +/** + * Rolls back assignment creation by deleting assignments and reverting user document updates. + * This is used when assignment creation fails to ensure atomicity. + * + * @param {string[]} userIds - Array of user IDs whose assignments should be rolled back. + * @param {string} administrationId - The administration ID. + * @param {IOrgsList} orgChunk - Optional org chunk to rollback stats for. + * @param {IAdministration} administrationData - Optional administration data for stats rollback. + */ +export const rollbackAssignmentCreation = async ( + userIds: string[], + administrationId: string, + orgChunk?: IOrgsList, + administrationData?: IAdministration +) => { + const db = getFirestore(); + + logger.warn(`Rolling back assignment creation for ${userIds.length} users`, { + administrationId, + userCount: userIds.length, + }); + + const batch = db.batch(); + let batchCount = 0; + const MAX_BATCH_SIZE = 500; + + for (const userId of userIds) { + if (batchCount >= MAX_BATCH_SIZE) { + await batch.commit(); + batchCount = 0; + } + + const assignmentRef = db + .collection("users") + .doc(userId) + .collection("assignments") + .doc(administrationId); + + batch.delete(assignmentRef); + batchCount++; + + const userDocRef = db.collection("users").doc(userId); + const fieldPathDate = new FieldPath( + "assignmentsAssigned", + administrationId + ); + const fieldPathList = new FieldPath("assignments", "assigned"); + + batch.update( + userDocRef, + fieldPathDate, + FieldValue.delete(), + fieldPathList, + FieldValue.arrayRemove(administrationId) + ); + batchCount++; + } + + if (batchCount > 0) { + await batch.commit(); + } + + // Rollback stats if org chunk and administration data are provided + if (orgChunk && administrationData && userIds.length > 0) { + try { + const { updateAdministrationStatsForOrgChunk } = await import( + "./on-assignment-updates.js" + ); + // Decrement stats by the number of users that were rolled back + await updateAdministrationStatsForOrgChunk( + administrationId, + orgChunk, + administrationData, + -userIds.length + ); + } catch (statsError: any) { + logger.error("Error rolling back administration stats", { + statsError, + administrationId, + userCount: userIds.length, + }); + } + } + + logger.info(`Successfully rolled back assignment creation`, { + administrationId, + userCount: userIds.length, + }); +}; + /** * Prepare a new assignment for a user. * @@ -301,6 +396,7 @@ export const addAssignmentToUsers = async ( transaction: Transaction ) => { console.log("hit addAssignmentToUsers"); + const db = getFirestore(); const assignments = await Promise.all( _map(users, (user) => prepareNewAssignment( @@ -312,11 +408,31 @@ export const addAssignmentToUsers = async ( ) ); - return _map(assignments, ([assignmentRef, assignmentData]) => { + return _map(assignments, ([assignmentRef, assignmentData], index: number) => { if (assignmentRef && assignmentData) { logger.debug(`Adding new assignment at ${assignmentRef.path}`, { assignmentSummary: summarizeAssignmentForLog(assignmentData), }); + + const userUid = users[index]; + if (userUid) { + const userDocRef = db.collection("users").doc(userUid); + const fieldPathDate = new FieldPath( + "assignmentsAssigned", + administrationId + ); + const fieldPathList = new FieldPath("assignments", "assigned"); + const dateAssigned = assignmentData.dateAssigned || new Date(); + + transaction.update( + userDocRef, + fieldPathDate, + dateAssigned, + fieldPathList, + FieldValue.arrayUnion(administrationId) + ); + } + return transaction.set(assignmentRef, assignmentData, { merge: true }); } else { return transaction; diff --git a/functions/levante-admin/src/assignments/on-assignment-updates.ts b/functions/levante-admin/src/assignments/on-assignment-updates.ts index 1b0c2d8..0537ca2 100644 --- a/functions/levante-admin/src/assignments/on-assignment-updates.ts +++ b/functions/levante-admin/src/assignments/on-assignment-updates.ts @@ -14,6 +14,7 @@ import type { DocumentDeletedEvent, DocumentUpdatedEvent, } from "../utils/utils.js"; +import type { IAdministration, IOrgsList } from "../interfaces.js"; type Status = "assigned" | "started" | "completed"; @@ -117,6 +118,59 @@ const incrementCompletionStatus = async ( } }; +/** + * Updates administration stats for assignments created for a given org chunk. + * This function is called synchronously to ensure stats are updated immediately. + * + * @param {string} administrationId - The administration ID. + * @param {IOrgsList} orgChunk - The org chunk that was processed. + * @param {IAdministration} administrationData - The administration data. + * @param {number} userCount - The number of users that were assigned (to increment stats by). + */ +export const updateAdministrationStatsForOrgChunk = async ( + administrationId: string, + orgChunk: IOrgsList, + administrationData: IAdministration, + userCount: number +) => { + if (userCount === 0) { + return; + } + + // Allow negative counts for rollback (decrementing stats) + const incrementBy = userCount; + + const db = getFirestore(); + const completionCollectionRef = db + .collection("administrations") + .doc(administrationId) + .collection("stats"); + + const orgList = _reduce( + orgChunk, + (acc: string[], value: string[]) => { + acc.push(...value); + return acc; + }, + [] + ); + orgList.push("total"); + + const taskIds = administrationData.assessments.map((a) => a.taskId); + + await db.runTransaction(async (transaction) => { + await incrementCompletionStatus( + orgList, + "assigned", + taskIds, + completionCollectionRef, + transaction, + userCount, + true + ); + }); +}; + /** * Event handler for when a new assignment document is created. * diff --git a/functions/levante-admin/src/assignments/sync-assignments.ts b/functions/levante-admin/src/assignments/sync-assignments.ts index d9152db..73cfd21 100644 --- a/functions/levante-admin/src/assignments/sync-assignments.ts +++ b/functions/levante-admin/src/assignments/sync-assignments.ts @@ -24,11 +24,13 @@ import { getUsersFromOrgs } from "../orgs/org-utils.js"; import { addAssignmentToUsers, updateAssignmentForUsers, + rollbackAssignmentCreation, } from "./assignment-utils.js"; import { summarizeIdListForLog, summarizeOrgsForLog, } from "../utils/logging.js"; +import { updateAdministrationStatsForOrgChunk } from "./on-assignment-updates.js"; /** * Sync globally defined adminstrations with user-specific assignments. @@ -191,85 +193,133 @@ export const updateAssignmentsForOrgChunkHandler = async ({ administrationData: IAdministration; orgChunk: IOrgsList; mode: "update" | "add"; -}) => { +}): Promise<{ success: boolean; userIds: string[]; error?: Error }> => { if (!["update", "add"].includes(mode)) { throw new Error(`Invalid mode: ${mode}. Expected 'update' or 'add'.`); } const db = getFirestore(); + const createdUserIds: string[] = []; - // Get all of the current users and update their assignments. The - // maximum number of docs we can update in a single transaction is - // ``MAX_TRANSACTIONS``. The number of affected users is potentially - // larger. So we loop through chunks of the userIds and update them in - // separate transactions if necessary. + try { + // Get all of the current users and update their assignments. The + // maximum number of docs we can update in a single transaction is + // ``MAX_TRANSACTIONS``. The number of affected users is potentially + // larger. So we loop through chunks of the userIds and update them in + // separate transactions if necessary. - // ``remainingUsers`` is a placeholder in the event that the number of - // affected users is greater than the maximum number of docs we can update - // in a single transaction. - let remainingUsers: string[] = []; + // ``remainingUsers`` is a placeholder in the event that the number of + // affected users is greater than the maximum number of docs we can update + // in a single transaction. + let remainingUsers: string[] = []; + let totalUsersAssigned = 0; - // Run the first transaction to get the user list - await db.runTransaction(async (transaction) => { - const usersToUpdate = await getUsersFromOrgs({ - orgs: orgChunk, - transaction, - includeArchived: false, // Do not assign updated assignment to archived users - }); + // Run the first transaction to get the user list + await db.runTransaction(async (transaction) => { + const usersToUpdate = await getUsersFromOrgs({ + orgs: orgChunk, + transaction, + includeArchived: false, // Do not assign updated assignment to archived users + }); - logger.info(`Updating assignment ${administrationId} for users`, { - orgChunkSummary: summarizeOrgsForLog(orgChunk), - userSummary: summarizeIdListForLog(usersToUpdate), + logger.info(`Updating assignment ${administrationId} for users`, { + orgChunkSummary: summarizeOrgsForLog(orgChunk), + userSummary: summarizeIdListForLog(usersToUpdate), + }); + + if (usersToUpdate.length !== 0) { + if (usersToUpdate.length <= MAX_TRANSACTIONS) { + // If the number of users is small enough, update them in this transaction. + if (mode === "update") { + return updateAssignmentForUsers( + usersToUpdate, + administrationId, + administrationData, + transaction + ); + } else { + console.log("adding assignments to users"); + totalUsersAssigned = usersToUpdate.length; + createdUserIds.push(...usersToUpdate); + return addAssignmentToUsers( + usersToUpdate, + administrationId, + administrationData, + transaction + ); + } + } else { + // Otherwise, just save for the next loop over user chunks. + remainingUsers = usersToUpdate; + return Promise.resolve(usersToUpdate.length); + } + } else { + return Promise.resolve(0); + } }); - if (usersToUpdate.length !== 0) { - if (usersToUpdate.length <= MAX_TRANSACTIONS) { - // If the number of users is small enough, update them in this transaction. + // If remainingUsersToRemove.length === 0, then these chunks will be of zero length + // and the entire loop below is a no-op. + for (const _userChunk of _chunk(remainingUsers, MAX_TRANSACTIONS)) { + await db.runTransaction(async (transaction) => { if (mode === "update") { return updateAssignmentForUsers( - usersToUpdate, + _userChunk, administrationId, administrationData, transaction ); } else { - console.log("adding assignments to users"); + totalUsersAssigned += _userChunk.length; + createdUserIds.push(..._userChunk); return addAssignmentToUsers( - usersToUpdate, + _userChunk, administrationId, administrationData, transaction ); } - } else { - // Otherwise, just save for the next loop over user chunks. - remainingUsers = usersToUpdate; - return Promise.resolve(usersToUpdate.length); - } - } else { - return Promise.resolve(0); + }); } - }); - // If remainingUsersToRemove.length === 0, then these chunks will be of zero length - // and the entire loop below is a no-op. - for (const _userChunk of _chunk(remainingUsers, MAX_TRANSACTIONS)) { - await db.runTransaction(async (transaction) => { - if (mode === "update") { - return updateAssignmentForUsers( - _userChunk, + // Update administration stats synchronously for immediate visibility + // Only update stats when adding new assignments (not when updating) + if (mode === "add" && totalUsersAssigned > 0) { + await updateAdministrationStatsForOrgChunk( + administrationId, + orgChunk, + administrationData, + totalUsersAssigned + ); + } + + return { success: true, userIds: createdUserIds }; + } catch (error: any) { + logger.error("Error creating assignments for org chunk", { + error, + administrationId, + orgChunkSummary: summarizeOrgsForLog(orgChunk), + createdUserIds, + }); + + // Rollback all created assignments if any failed + if (mode === "add" && createdUserIds.length > 0) { + try { + await rollbackAssignmentCreation( + createdUserIds, administrationId, - administrationData, - transaction + orgChunk, + administrationData ); - } else { - return addAssignmentToUsers( - _userChunk, + } catch (rollbackError: any) { + logger.error("Error during rollback of assignment creation", { + rollbackError, administrationId, - administrationData, - transaction - ); + userIds: createdUserIds, + }); } - }); + } + + return { success: false, userIds: createdUserIds, error }; } }; diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index 50f0952..05a147b 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -82,26 +82,94 @@ const syncNewAdministrationAssignments = async ( const orgChunks = chunkOrgs(minimalOrgs, 100); const maxChunksToProcessSync = 3; + const allCreatedUserIds: string[] = []; + const processedChunks: Array<{ orgChunk: IOrgsList; userIds: string[] }> = []; - for (let i = 0; i < Math.min(orgChunks.length, maxChunksToProcessSync); i++) { - const orgChunk = orgChunks[i]; - await updateAssignmentsForOrgChunkHandler({ + try { + for ( + let i = 0; + i < Math.min(orgChunks.length, maxChunksToProcessSync); + i++ + ) { + const orgChunk = orgChunks[i]; + const result = await updateAssignmentsForOrgChunkHandler({ + administrationId, + administrationData: currData, + orgChunk, + mode: "add", + }); + + if (!result.success) { + // If this chunk failed, rollback all previous chunks + logger.error( + `Assignment creation failed for org chunk ${ + i + 1 + }. Rolling back all previous chunks.`, + { + administrationId, + chunkIndex: i, + error: result.error, + totalChunksProcessed: processedChunks.length, + } + ); + + // Rollback all previously created assignments + if (allCreatedUserIds.length > 0) { + const { rollbackAssignmentCreation } = await import( + "./assignments/assignment-utils.js" + ); + await rollbackAssignmentCreation(allCreatedUserIds, administrationId); + } + + throw new Error( + `Failed to create assignment for all participants. ${ + result.error?.message || "Unknown error" + }. Please try again.` + ); + } + + // Track successful chunk + allCreatedUserIds.push(...result.userIds); + processedChunks.push({ orgChunk, userIds: result.userIds }); + } + + if (orgChunks.length > maxChunksToProcessSync) { + logger.info( + `Processed first ${maxChunksToProcessSync} org chunks synchronously. Remaining chunks will be processed by background trigger.`, + { + administrationId, + totalChunks: orgChunks.length, + processedChunks: maxChunksToProcessSync, + totalUsersAssigned: allCreatedUserIds.length, + } + ); + } + + logger.info(`Successfully created assignments for all participants`, { administrationId, - administrationData: currData, - orgChunk, - mode: "add", + totalUsersAssigned: allCreatedUserIds.length, + chunksProcessed: processedChunks.length, }); - } - - if (orgChunks.length > maxChunksToProcessSync) { - logger.info( - `Processed first ${maxChunksToProcessSync} org chunks synchronously. Remaining chunks will be processed by background trigger.`, - { - administrationId, - totalChunks: orgChunks.length, - processedChunks: maxChunksToProcessSync, + } catch (error: any) { + // If we get here, either a chunk failed or rollback failed + // Try to rollback any remaining assignments + if (allCreatedUserIds.length > 0) { + try { + const { rollbackAssignmentCreation } = await import( + "./assignments/assignment-utils.js" + ); + await rollbackAssignmentCreation(allCreatedUserIds, administrationId); + } catch (rollbackError: any) { + logger.error("Error during final rollback", { + rollbackError, + administrationId, + userIds: allCreatedUserIds, + }); } - ); + } + + // Re-throw the error so it can be caught by the handler + throw error; } }; From 8d1b317b25786184b74fa67d9fccad32f336db40 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 12 Nov 2025 10:39:35 -0300 Subject: [PATCH 05/15] deleting the new assignment doc if the process is reverted --- .../src/assignments/assignment-utils.ts | 54 +++++++++++++++ .../levante-admin/src/upsertAdministration.ts | 67 +++++++++++++++++-- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/functions/levante-admin/src/assignments/assignment-utils.ts b/functions/levante-admin/src/assignments/assignment-utils.ts index 3f75c1b..eeae81f 100644 --- a/functions/levante-admin/src/assignments/assignment-utils.ts +++ b/functions/levante-admin/src/assignments/assignment-utils.ts @@ -120,6 +120,60 @@ export const removeAssignmentFromUsers = async ( ); }; +/** + * Rolls back administration document creation by deleting the administration document + * and removing it from the creator's administrationsCreated array. + * This is used when assignment creation fails for a new administration to ensure full atomicity. + * + * @param {string} administrationId - The administration ID to rollback. + * @param {string} creatorUid - The UID of the user who created the administration. + */ +export const rollbackAdministrationCreation = async ( + administrationId: string, + creatorUid: string +) => { + const db = getFirestore(); + + logger.warn(`Rolling back administration document creation`, { + administrationId, + creatorUid, + }); + + try { + const administrationDocRef = db + .collection("administrations") + .doc(administrationId); + + const creatorDocRef = db.collection("users").doc(creatorUid); + + await db.runTransaction(async (transaction) => { + const adminDoc = await transaction.get(administrationDocRef); + const creatorDoc = await transaction.get(creatorDocRef); + + if (adminDoc.exists) { + transaction.delete(administrationDocRef); + } + + if (creatorDoc.exists) { + const fieldPath = new FieldPath("adminData", "administrationsCreated"); + transaction.update(creatorDocRef, fieldPath, FieldValue.arrayRemove(administrationId)); + } + }); + + logger.info(`Successfully rolled back administration document creation`, { + administrationId, + creatorUid, + }); + } catch (error: any) { + logger.error("Error rolling back administration document creation", { + error, + administrationId, + creatorUid, + }); + throw error; + } +}; + /** * Rolls back assignment creation by deleting assignments and reverting user document updates. * This is used when assignment creation fails to ensure atomicity. diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index 05a147b..f846d39 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -70,7 +70,9 @@ interface IAdministrationDoc { const syncNewAdministrationAssignments = async ( administrationId: string, administrationDocRef: DocumentReference, - currData: IAdministration + currData: IAdministration, + creatorUid?: string, + isNewAdministration: boolean = false ) => { const { minimalOrgs } = await standardizeAdministrationOrgs({ administrationId, @@ -121,6 +123,14 @@ const syncNewAdministrationAssignments = async ( await rollbackAssignmentCreation(allCreatedUserIds, administrationId); } + // If this is a new administration, also rollback the administration document + if (isNewAdministration && creatorUid) { + const { rollbackAdministrationCreation } = await import( + "./assignments/assignment-utils.js" + ); + await rollbackAdministrationCreation(administrationId, creatorUid); + } + throw new Error( `Failed to create assignment for all participants. ${ result.error?.message || "Unknown error" @@ -160,7 +170,7 @@ const syncNewAdministrationAssignments = async ( ); await rollbackAssignmentCreation(allCreatedUserIds, administrationId); } catch (rollbackError: any) { - logger.error("Error during final rollback", { + logger.error("Error during final rollback of assignments", { rollbackError, administrationId, userIds: allCreatedUserIds, @@ -168,6 +178,22 @@ const syncNewAdministrationAssignments = async ( } } + // If this is a new administration, also rollback the administration document + if (isNewAdministration && creatorUid) { + try { + const { rollbackAdministrationCreation } = await import( + "./assignments/assignment-utils.js" + ); + await rollbackAdministrationCreation(administrationId, creatorUid); + } catch (adminRollbackError: any) { + logger.error("Error during final rollback of administration document", { + adminRollbackError, + administrationId, + creatorUid, + }); + } + } + // Re-throw the error so it can be caught by the handler throw error; } @@ -627,12 +653,25 @@ export const upsertAdministrationHandler = async ( const administrationData = administrationDoc.data() as IAdministration; const isNewAdministration = !administrationId; + const creatorUid = administrationData?.createdBy; if (isNewAdministration) { + if (!creatorUid) { + logger.error( + `Cannot sync assignments: administration ${newAdministrationId} has no createdBy field`, + { administrationId: newAdministrationId } + ); + throw new HttpsError( + "internal", + "Administration document is missing creator information" + ); + } await syncNewAdministrationAssignments( newAdministrationId, administrationDocRef, - administrationData + administrationData, + creatorUid, + true ); } else if (prevData) { await syncModifiedAdministrationAssignments( @@ -642,10 +681,22 @@ export const upsertAdministrationHandler = async ( administrationData ); } else { + if (!creatorUid) { + logger.error( + `Cannot sync assignments: administration ${newAdministrationId} has no createdBy field`, + { administrationId: newAdministrationId } + ); + throw new HttpsError( + "internal", + "Administration document is missing creator information" + ); + } await syncNewAdministrationAssignments( newAdministrationId, administrationDocRef, - administrationData + administrationData, + creatorUid, + true ); } @@ -657,6 +708,14 @@ export const upsertAdministrationHandler = async ( error: syncError, administrationId: newAdministrationId, }); + + // If this is a new administration and sync failed, the error was already thrown + // and should have triggered rollback. Re-throw to prevent the function from + // returning successfully when assignments failed to be created. + const isNewAdministration = !administrationId; + if (isNewAdministration) { + throw syncError; + } } return { status: "ok", administrationId: newAdministrationId }; From 0ee73ee1ad7412b8a41c734aa824fffeb98059bb Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 12 Nov 2025 10:39:49 -0300 Subject: [PATCH 06/15] updated README file --- .../SYNCHRONOUS_ASSIGNMENT_SYNC.md | 100 +++++++++++++++--- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md index ccd21b8..543c2e6 100644 --- a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md +++ b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md @@ -54,10 +54,12 @@ const syncNewAdministrationAssignments = async ( ``` **Process:** + 1. Standardizes administration orgs using `standardizeAdministrationOrgs` 2. Chunks orgs into groups of 100 3. Processes the first 3 chunks synchronously using `updateAssignmentsForOrgChunkHandler` -4. Logs if additional chunks will be processed by the background trigger +4. Tracks all successfully created assignments for rollback if any chunk fails +5. Logs if additional chunks will be processed by the background trigger #### `syncModifiedAdministrationAssignments` @@ -73,6 +75,7 @@ const syncModifiedAdministrationAssignments = async ( ``` **Process:** + 1. Calculates removed orgs by comparing previous and current org lists 2. Removes assignments from users in removed orgs 3. Standardizes administration orgs @@ -81,6 +84,7 @@ const syncModifiedAdministrationAssignments = async ( ### Modified Flow **Before:** + ``` upsertAdministration → Transaction → Return ↓ @@ -88,6 +92,7 @@ upsertAdministration → Transaction → Return ``` **After:** + ``` upsertAdministration → Transaction → Sync Assignments (sync) → Return ↓ @@ -124,26 +129,89 @@ The implementation processes the first **3 org chunks** synchronously, where eac - **Performance**: Keeps response times reasonable These limits can be adjusted by modifying the `maxChunksToProcessSync` constant in: + - `syncNewAdministrationAssignments` - `syncModifiedAdministrationAssignments` -## Error Handling +## Transactional Guarantees + +The assignment creation process implements **all-or-nothing semantics** to ensure data integrity: + +### All-or-Nothing Behavior + +When creating assignments for participants, the system guarantees that **all participants receive the assignment, or none do**. This prevents partial assignment states where some participants have assignments while others do not. + +### Rollback Mechanism + +If assignment creation fails for **any** participant in any chunk: + +1. **Automatic Rollback**: The system automatically calls `rollbackAssignmentCreation` to revert all assignment-related changes +2. **Assignment Deletion**: All assignment documents created during the process are deleted +3. **User Document Reversion**: The administration ID is removed from user documents' `assignments` arrays +4. **Statistics Reversion**: Any administration statistics that were incremented are decremented +5. **Administration Document Rollback** (for new administrations only): If this is a new administration, the system also calls `rollbackAdministrationCreation` to: + - Delete the administration document from the `administrations` collection + - Remove the administration ID from the creator's `adminData.administrationsCreated` array +6. **Error Propagation**: The error is thrown, causing the entire `upsertAdministration` operation to fail + +### Rollback Implementation + +The rollback process consists of two functions: -The synchronous sync is wrapped in a try-catch block that: +**`rollbackAssignmentCreation`** (in `assignment-utils.ts`): -- Logs errors without failing the main `upsertAdministration` operation -- Allows the function to return successfully even if sync fails -- Relies on the background trigger as a fallback for error recovery +- Processes rollbacks in batches of up to 500 operations for efficiency +- Deletes assignment documents from the `users/{userId}/assignments/{administrationId}` subcollection +- Removes the administration ID from user documents' `assignments.assigned` array +- Deletes the `assignmentsAssigned.{administrationId}` timestamp field +- Decrements administration statistics if org chunk and administration data are provided -This ensures that assignment creation/updates are not blocked by sync errors. Even if the synchronous sync encounters an error, the administration document is still created/updated successfully, and the background trigger will process the assignments asynchronously. +**`rollbackAdministrationCreation`** (in `assignment-utils.ts`): + +- Used only for new administrations when assignment creation fails +- Deletes the administration document from the `administrations` collection +- Removes the administration ID from the creator's `adminData.administrationsCreated` array +- Uses a Firestore transaction to ensure atomicity + +### Chunk-Level Rollback + +The system tracks all successfully created assignments across chunks: + +- Each chunk's successful assignments are tracked in `allCreatedUserIds` +- If any chunk fails, all previously processed chunks are rolled back +- This ensures atomicity across the entire synchronous processing phase + +### Error Handling + +The synchronous sync implements comprehensive error handling: + +- **Chunk-Level Errors**: If `updateAssignmentsForOrgChunkHandler` returns `success: false`, all previous chunks are rolled back and an error is thrown +- **User-Level Errors**: If assignment creation fails for any user within a chunk, that entire chunk is rolled back and an error is thrown +- **Rollback Errors**: If rollback itself fails, errors are logged but the original error is still propagated +- **Final Rollback**: A catch block in `syncNewAdministrationAssignments` ensures rollback is attempted even if the error occurs outside the chunk loop +- **Error Propagation**: Errors thrown from `syncNewAdministrationAssignments` are caught by the try-catch block in `upsertAdministrationHandler`. For new administrations, the error is re-thrown to prevent the function from returning successfully. For updates, the error is logged but not re-thrown, allowing the function to return successfully. ### Error Recovery -If synchronous sync fails: -1. The error is logged with full context -2. The `upsertAdministration` function still returns successfully -3. The background Firestore trigger will process all assignments -4. Users may experience a slight delay, but assignments will appear eventually +If synchronous sync fails for a **new administration**: + +1. All created assignments are rolled back automatically via `rollbackAssignmentCreation` +2. The administration document is deleted via `rollbackAdministrationCreation` +3. The administration ID is removed from the creator's `adminData.administrationsCreated` array +4. The error is logged with full context (chunk index, user IDs, administration ID, creator UID) +5. The error is re-thrown, causing the `upsertAdministration` function to fail +6. The client receives an error message: "Failed to create assignment for all participants. Please try again." +7. The entire operation is atomic - either everything succeeds or everything is rolled back + +If synchronous sync fails for an **existing administration update**: + +1. All created assignments are rolled back automatically via `rollbackAssignmentCreation` +2. The error is logged but not re-thrown +3. The `upsertAdministration` function returns successfully (the administration document update remains) +4. The background Firestore trigger will process assignments asynchronously as a fallback +5. Users may experience a delay, but assignments will eventually appear via the background trigger + +**Note**: For new administrations, the rollback is complete and atomic - both assignments and the administration document are removed. For updates, only assignments are rolled back since the administration document update is considered successful even if assignment sync fails. ## Performance Considerations @@ -155,6 +223,7 @@ If synchronous sync fails: ### Transaction Limits The implementation respects Firestore transaction limits: + - Maximum of `MAX_TRANSACTIONS` (100) documents per transaction - Large user sets are processed in chunks across multiple transactions @@ -178,8 +247,10 @@ Potential enhancements: ## Related Files -- `functions/levante-admin/src/upsertAdministration.ts` - Main handler with synchronous sync -- `functions/levante-admin/src/assignments/sync-assignments.ts` - Assignment sync utilities +- `functions/levante-admin/src/upsertAdministration.ts` - Main handler with synchronous sync and rollback logic +- `functions/levante-admin/src/assignments/sync-assignments.ts` - Assignment sync utilities with error handling +- `functions/levante-admin/src/assignments/assignment-utils.ts` - Contains `rollbackAssignmentCreation` and `rollbackAdministrationCreation` functions +- `functions/levante-admin/src/assignments/on-assignment-updates.ts` - Administration stats management - `functions/levante-admin/src/administrations/sync-administrations.ts` - Background sync handlers - `functions/levante-admin/src/index.ts` - Firestore trigger definitions @@ -190,4 +261,3 @@ Potential enhancements: - The synchronous processing uses the same underlying functions as the background trigger for consistency - The helper functions (`syncNewAdministrationAssignments` and `syncModifiedAdministrationAssignments`) are defined as private functions within the `upsertAdministration.ts` file - For update operations, the previous administration data is fetched before the transaction to enable proper comparison of org changes - From 90f3226aeac222b0a63c485a4f6d8d17ed13a6d9 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Thu, 13 Nov 2025 14:36:41 -0300 Subject: [PATCH 07/15] removed org chunk limit to work on all of them --- .../levante-admin/src/upsertAdministration.ts | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index f846d39..3f18568 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -83,16 +83,11 @@ const syncNewAdministrationAssignments = async ( }); const orgChunks = chunkOrgs(minimalOrgs, 100); - const maxChunksToProcessSync = 3; const allCreatedUserIds: string[] = []; const processedChunks: Array<{ orgChunk: IOrgsList; userIds: string[] }> = []; try { - for ( - let i = 0; - i < Math.min(orgChunks.length, maxChunksToProcessSync); - i++ - ) { + for (let i = 0; i < orgChunks.length; i++) { const orgChunk = orgChunks[i]; const result = await updateAssignmentsForOrgChunkHandler({ administrationId, @@ -143,22 +138,11 @@ const syncNewAdministrationAssignments = async ( processedChunks.push({ orgChunk, userIds: result.userIds }); } - if (orgChunks.length > maxChunksToProcessSync) { - logger.info( - `Processed first ${maxChunksToProcessSync} org chunks synchronously. Remaining chunks will be processed by background trigger.`, - { - administrationId, - totalChunks: orgChunks.length, - processedChunks: maxChunksToProcessSync, - totalUsersAssigned: allCreatedUserIds.length, - } - ); - } - logger.info(`Successfully created assignments for all participants`, { administrationId, totalUsersAssigned: allCreatedUserIds.length, chunksProcessed: processedChunks.length, + totalChunks: orgChunks.length, }); } catch (error: any) { // If we get here, either a chunk failed or rollback failed @@ -280,28 +264,34 @@ const syncModifiedAdministrationAssignments = async ( }); const orgChunks = chunkOrgs(minimalOrgs, 100); - const maxChunksToProcessSync = 3; - for (let i = 0; i < Math.min(orgChunks.length, maxChunksToProcessSync); i++) { + for (let i = 0; i < orgChunks.length; i++) { const orgChunk = orgChunks[i]; - await updateAssignmentsForOrgChunkHandler({ + const result = await updateAssignmentsForOrgChunkHandler({ administrationId, administrationData: currData, orgChunk, mode: "update", }); - } - if (orgChunks.length > maxChunksToProcessSync) { - logger.info( - `Processed first ${maxChunksToProcessSync} org chunks synchronously. Remaining chunks will be processed by background trigger.`, - { + if (!result.success) { + logger.error(`Assignment update failed for org chunk ${i + 1}.`, { administrationId, - totalChunks: orgChunks.length, - processedChunks: maxChunksToProcessSync, - } - ); + chunkIndex: i, + error: result.error, + }); + throw new Error( + `Failed to update assignment for all participants. ${ + result.error?.message || "Unknown error" + }. Please try again.` + ); + } } + + logger.info(`Successfully updated assignments for all participants`, { + administrationId, + chunksProcessed: orgChunks.length, + }); }; export const upsertAdministrationHandler = async ( From 96784a542b02df8fc7c28eb9576aa29d69b57fb6 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Thu, 13 Nov 2025 14:37:00 -0300 Subject: [PATCH 08/15] updated the README file to match the new sync process --- .../SYNCHRONOUS_ASSIGNMENT_SYNC.md | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md index 543c2e6..7a765f5 100644 --- a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md +++ b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md @@ -16,19 +16,16 @@ When a user created or edited an assignment via `upsertAdministration`: ## Solution -The solution implements **synchronous assignment processing** within the `upsertAdministration` handler, ensuring that critical assignment operations complete before the function returns. This provides immediate visibility of assignments while maintaining the background trigger for eventual consistency. +The solution implements **fully synchronous and atomic assignment processing** within the `upsertAdministration` handler. **All assignment operations are processed synchronously** before the function returns, ensuring immediate visibility of assignments. The system also implements comprehensive atomic rollback - if any participant fails to receive an assignment, the entire operation is rolled back, including the administration document for new administrations. ## Implementation Details ### Architecture -The implementation follows a two-tier approach: +The implementation processes **all org chunks synchronously** within the `upsertAdministration` handler: -1. **Synchronous Processing (Primary)**: Processes the first 3 org chunks (up to ~300 organizations) immediately -2. **Background Processing (Secondary)**: The existing Firestore trigger continues to handle: - - Remaining org chunks for large assignments - - Eventual consistency - - Edge cases and retries +1. **Synchronous Processing**: All org chunks are processed synchronously before the function returns +2. **Background Processing (Redundant)**: The existing Firestore trigger still fires and processes chunks asynchronously, but this is redundant since all chunks are already processed synchronously. The background processing is safe due to idempotent operations but adds unnecessary overhead. ### Code Structure @@ -49,7 +46,9 @@ Handles synchronous assignment creation for new administrations: const syncNewAdministrationAssignments = async ( administrationId: string, administrationDocRef: DocumentReference, - currData: IAdministration + currData: IAdministration, + creatorUid?: string, + isNewAdministration: boolean = false ) ``` @@ -57,9 +56,11 @@ const syncNewAdministrationAssignments = async ( 1. Standardizes administration orgs using `standardizeAdministrationOrgs` 2. Chunks orgs into groups of 100 -3. Processes the first 3 chunks synchronously using `updateAssignmentsForOrgChunkHandler` -4. Tracks all successfully created assignments for rollback if any chunk fails -5. Logs if additional chunks will be processed by the background trigger +3. Processes **all chunks synchronously** using `updateAssignmentsForOrgChunkHandler` +4. Tracks all successfully created assignments in `allCreatedUserIds` for rollback if any chunk fails +5. If any chunk fails, immediately rolls back all previously processed chunks +6. If this is a new administration and any chunk fails, also rolls back the administration document +7. Re-throws errors to ensure the function fails if assignment creation fails #### `syncModifiedAdministrationAssignments` @@ -79,7 +80,8 @@ const syncModifiedAdministrationAssignments = async ( 1. Calculates removed orgs by comparing previous and current org lists 2. Removes assignments from users in removed orgs 3. Standardizes administration orgs -4. Processes the first 3 org chunks synchronously for updates +4. Processes **all org chunks synchronously** for updates +5. If any chunk fails, throws an error (errors are caught by the handler and logged but not re-thrown for updates) ### Modified Flow @@ -94,44 +96,42 @@ upsertAdministration → Transaction → Return **After:** ``` -upsertAdministration → Transaction → Sync Assignments (sync) → Return +upsertAdministration → Transaction → Sync All Assignments (sync, atomic) → Return ↓ - Firestore Trigger (async) → Process Remaining Chunks + Firestore Trigger (async) → Process All Chunks (redundant, idempotent) ``` +The synchronous sync is **fully atomic** - if any participant fails to receive an assignment, the entire operation (including the administration document for new administrations) is rolled back before the function returns. All chunks are processed synchronously, so the background trigger is redundant but safe due to idempotent operations. + ### Implementation Details The synchronous sync happens immediately after the transaction commits: 1. The administration document is fetched from Firestore (to ensure server timestamps are populated) 2. Based on whether it's a create or update operation, the appropriate sync function is called -3. The sync function processes org chunks synchronously -4. The function returns, allowing the client to see the assignment immediately -5. The background trigger continues processing any remaining chunks asynchronously +3. The sync function processes **all org chunks synchronously** +4. The function returns, allowing the client to see all assignments immediately +5. The background trigger still fires but is redundant since all chunks are already processed ### Integration with Background Trigger -The background Firestore trigger (`syncAssignmentsOnAdministrationUpdate`) continues to operate: +The background Firestore trigger (`syncAssignmentsOnAdministrationUpdate`) still fires after document writes, but it is **redundant** since all chunks are already processed synchronously: - **Idempotent Operations**: All assignment operations are idempotent, so duplicate processing is safe -- **Large Scale**: For assignments with more than 3 org chunks, the background trigger processes remaining chunks -- **Resilience**: Provides backup processing if synchronous processing encounters errors -- **Eventual Consistency**: Ensures all assignments are eventually processed correctly +- **Redundant Processing**: The background trigger processes all chunks again, which is unnecessary overhead +- **Future Optimization**: Consider disabling or skipping the background trigger when all chunks have been processed synchronously to reduce unnecessary processing ## Configuration -### Synchronous Processing Limits - -The implementation processes the first **3 org chunks** synchronously, where each chunk contains up to **100 organizations**. This balances: +### Synchronous Processing -- **Immediate Visibility**: Users see assignments immediately for most common scenarios -- **Function Timeout**: Prevents Cloud Functions from timing out on very large assignments -- **Performance**: Keeps response times reasonable +The implementation processes **all org chunks synchronously**, where each chunk contains up to **100 organizations**. This ensures: -These limits can be adjusted by modifying the `maxChunksToProcessSync` constant in: +- **Immediate Visibility**: Users see all assignments immediately, regardless of scale +- **Complete Atomicity**: All participants receive assignments or none do (for new administrations) +- **No Partial States**: Eliminates the need for background processing to complete assignments -- `syncNewAdministrationAssignments` -- `syncModifiedAdministrationAssignments` +**Note**: For very large assignments (hundreds of chunks), this may approach Cloud Functions timeout limits. Monitor function execution times and consider implementing chunk processing limits if timeout issues occur. ## Transactional Guarantees @@ -217,8 +217,9 @@ If synchronous sync fails for an **existing administration update**: ### Function Execution Time -- **Small Assignments** (≤3 chunks): Fully processed synchronously -- **Large Assignments** (>3 chunks): First 3 chunks processed synchronously, remainder processed in background +- **All Assignments**: All chunks are processed synchronously before the function returns +- **Execution Time**: Scales linearly with the number of org chunks (each chunk processes up to 100 organizations) +- **Timeout Considerations**: Cloud Functions have a maximum timeout (typically 540 seconds for 2nd gen). For very large assignments, monitor execution time to ensure it stays within limits ### Transaction Limits @@ -240,10 +241,10 @@ When testing this functionality: Potential enhancements: -1. **Configurable Chunk Limits**: Make `maxChunksToProcessSync` configurable via environment variables -2. **Progress Tracking**: Add metadata to track which chunks have been processed synchronously -3. **Skip Background Processing**: Optionally skip background trigger for fully-synced assignments -4. **Metrics**: Track synchronous vs. background processing ratios +1. **Skip Background Processing**: Disable or skip the background trigger when all chunks have been processed synchronously to reduce unnecessary overhead +2. **Progress Tracking**: Add metadata to track which chunks have been processed synchronously (useful if we need to reintroduce chunk limits) +3. **Configurable Chunk Limits**: If timeout issues occur, make chunk processing limits configurable via environment variables +4. **Metrics**: Track synchronous processing performance and identify any timeout issues ## Related Files @@ -256,8 +257,13 @@ Potential enhancements: ## Notes -- The background trigger (`syncAssignmentsOnAdministrationUpdate`) will still fire after document writes -- Duplicate processing is safe due to idempotent operations +- The background trigger (`syncAssignmentsOnAdministrationUpdate`) will still fire after document writes, but it is redundant since all chunks are already processed synchronously +- Duplicate processing is safe due to idempotent operations, but adds unnecessary overhead - The synchronous processing uses the same underlying functions as the background trigger for consistency - The helper functions (`syncNewAdministrationAssignments` and `syncModifiedAdministrationAssignments`) are defined as private functions within the `upsertAdministration.ts` file - For update operations, the previous administration data is fetched before the transaction to enable proper comparison of org changes +- The rollback mechanism ensures **complete atomicity** - if any participant fails to receive an assignment, the entire operation (including the administration document for new administrations) is rolled back +- Rollback happens at the chunk level - if any chunk fails, all previously processed chunks are rolled back +- The `updateAssignmentsForOrgChunkHandler` function also implements its own rollback for failed chunks, providing defense-in-depth +- Statistics are updated synchronously when adding new assignments (mode: "add") to ensure immediate visibility +- **All chunks are processed synchronously** - there is no limit on the number of chunks processed, ensuring complete assignment creation before the function returns From 71c26a1906141de476422dae835383ae743d775c Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Thu, 13 Nov 2025 14:40:24 -0300 Subject: [PATCH 09/15] renamed syncNewAdministrationAssignments and syncModifiedAdministrationAssignments to createAssignments and updateAssignments respectively --- .../SYNCHRONOUS_ASSIGNMENT_SYNC.md | 18 +++++++++--------- .../levante-admin/src/upsertAdministration.ts | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md index 7a765f5..0b270f3 100644 --- a/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md +++ b/functions/levante-admin/SYNCHRONOUS_ASSIGNMENT_SYNC.md @@ -31,19 +31,19 @@ The implementation processes **all org chunks synchronously** within the `upsert The synchronous sync logic is implemented as two helper functions within `upsertAdministration.ts`: -- `syncNewAdministrationAssignments` - Handles new administration creation -- `syncModifiedAdministrationAssignments` - Handles administration updates +- `createAssignments` - Handles new administration creation +- `updateAssignments` - Handles administration updates These functions are called directly after the Firestore transaction completes, ensuring assignments are processed before the function returns to the client. ### Key Functions -#### `syncNewAdministrationAssignments` +#### `createAssignments` Handles synchronous assignment creation for new administrations: ```typescript -const syncNewAdministrationAssignments = async ( +const createAssignments = async ( administrationId: string, administrationDocRef: DocumentReference, currData: IAdministration, @@ -62,12 +62,12 @@ const syncNewAdministrationAssignments = async ( 6. If this is a new administration and any chunk fails, also rolls back the administration document 7. Re-throws errors to ensure the function fails if assignment creation fails -#### `syncModifiedAdministrationAssignments` +#### `updateAssignments` Handles synchronous assignment updates for modified administrations: ```typescript -const syncModifiedAdministrationAssignments = async ( +const updateAssignments = async ( administrationId: string, administrationDocRef: DocumentReference, prevData: IAdministration, @@ -188,8 +188,8 @@ The synchronous sync implements comprehensive error handling: - **Chunk-Level Errors**: If `updateAssignmentsForOrgChunkHandler` returns `success: false`, all previous chunks are rolled back and an error is thrown - **User-Level Errors**: If assignment creation fails for any user within a chunk, that entire chunk is rolled back and an error is thrown - **Rollback Errors**: If rollback itself fails, errors are logged but the original error is still propagated -- **Final Rollback**: A catch block in `syncNewAdministrationAssignments` ensures rollback is attempted even if the error occurs outside the chunk loop -- **Error Propagation**: Errors thrown from `syncNewAdministrationAssignments` are caught by the try-catch block in `upsertAdministrationHandler`. For new administrations, the error is re-thrown to prevent the function from returning successfully. For updates, the error is logged but not re-thrown, allowing the function to return successfully. +- **Final Rollback**: A catch block in `createAssignments` ensures rollback is attempted even if the error occurs outside the chunk loop +- **Error Propagation**: Errors thrown from `createAssignments` are caught by the try-catch block in `upsertAdministrationHandler`. For new administrations, the error is re-thrown to prevent the function from returning successfully. For updates, the error is logged but not re-thrown, allowing the function to return successfully. ### Error Recovery @@ -260,7 +260,7 @@ Potential enhancements: - The background trigger (`syncAssignmentsOnAdministrationUpdate`) will still fire after document writes, but it is redundant since all chunks are already processed synchronously - Duplicate processing is safe due to idempotent operations, but adds unnecessary overhead - The synchronous processing uses the same underlying functions as the background trigger for consistency -- The helper functions (`syncNewAdministrationAssignments` and `syncModifiedAdministrationAssignments`) are defined as private functions within the `upsertAdministration.ts` file +- The helper functions (`createAssignments` and `updateAssignments`) are defined as private functions within the `upsertAdministration.ts` file - For update operations, the previous administration data is fetched before the transaction to enable proper comparison of org changes - The rollback mechanism ensures **complete atomicity** - if any participant fails to receive an assignment, the entire operation (including the administration document for new administrations) is rolled back - Rollback happens at the chunk level - if any chunk fails, all previously processed chunks are rolled back diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index 3f18568..5eabb2d 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -67,7 +67,7 @@ interface IAdministrationDoc { creatorName: string; } -const syncNewAdministrationAssignments = async ( +const createAssignments = async ( administrationId: string, administrationDocRef: DocumentReference, currData: IAdministration, @@ -183,7 +183,7 @@ const syncNewAdministrationAssignments = async ( } }; -const syncModifiedAdministrationAssignments = async ( +const updateAssignments = async ( administrationId: string, administrationDocRef: DocumentReference, prevData: IAdministration, @@ -656,7 +656,7 @@ export const upsertAdministrationHandler = async ( "Administration document is missing creator information" ); } - await syncNewAdministrationAssignments( + await createAssignments( newAdministrationId, administrationDocRef, administrationData, @@ -664,7 +664,7 @@ export const upsertAdministrationHandler = async ( true ); } else if (prevData) { - await syncModifiedAdministrationAssignments( + await updateAssignments( newAdministrationId, administrationDocRef, prevData, @@ -681,7 +681,7 @@ export const upsertAdministrationHandler = async ( "Administration document is missing creator information" ); } - await syncNewAdministrationAssignments( + await createAssignments( newAdministrationId, administrationDocRef, administrationData, From 4fa0e8bf0bdd5a937ed0d23a78279a21c500d2a5 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Tue, 2 Dec 2025 11:52:32 -0300 Subject: [PATCH 10/15] moved imports to the top of the file --- .../levante-admin/src/upsertAdministration.ts | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index 5eabb2d..242884f 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -22,7 +22,11 @@ import { getExhaustiveOrgs, getUsersFromOrgs, } from "./orgs/org-utils.js"; -import { removeOrgsFromAssignments } from "./assignments/assignment-utils.js"; +import { + removeOrgsFromAssignments, + rollbackAssignmentCreation, + rollbackAdministrationCreation, +} from "./assignments/assignment-utils.js"; import _chunk from "lodash-es/chunk.js"; import { MAX_TRANSACTIONS } from "./utils/utils.js"; @@ -112,17 +116,11 @@ const createAssignments = async ( // Rollback all previously created assignments if (allCreatedUserIds.length > 0) { - const { rollbackAssignmentCreation } = await import( - "./assignments/assignment-utils.js" - ); await rollbackAssignmentCreation(allCreatedUserIds, administrationId); } // If this is a new administration, also rollback the administration document if (isNewAdministration && creatorUid) { - const { rollbackAdministrationCreation } = await import( - "./assignments/assignment-utils.js" - ); await rollbackAdministrationCreation(administrationId, creatorUid); } @@ -149,9 +147,6 @@ const createAssignments = async ( // Try to rollback any remaining assignments if (allCreatedUserIds.length > 0) { try { - const { rollbackAssignmentCreation } = await import( - "./assignments/assignment-utils.js" - ); await rollbackAssignmentCreation(allCreatedUserIds, administrationId); } catch (rollbackError: any) { logger.error("Error during final rollback of assignments", { @@ -165,9 +160,6 @@ const createAssignments = async ( // If this is a new administration, also rollback the administration document if (isNewAdministration && creatorUid) { try { - const { rollbackAdministrationCreation } = await import( - "./assignments/assignment-utils.js" - ); await rollbackAdministrationCreation(administrationId, creatorUid); } catch (adminRollbackError: any) { logger.error("Error during final rollback of administration document", { From 336d4e11e06adc3d5b9f6a4fb8635119082dc329 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Tue, 2 Dec 2025 11:53:04 -0300 Subject: [PATCH 11/15] removed redundant chunk of code --- .../src/assignments/sync-assignments.ts | 42 ++----------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/functions/levante-admin/src/assignments/sync-assignments.ts b/functions/levante-admin/src/assignments/sync-assignments.ts index 73cfd21..762bba0 100644 --- a/functions/levante-admin/src/assignments/sync-assignments.ts +++ b/functions/levante-admin/src/assignments/sync-assignments.ts @@ -207,16 +207,12 @@ export const updateAssignmentsForOrgChunkHandler = async ({ // ``MAX_TRANSACTIONS``. The number of affected users is potentially // larger. So we loop through chunks of the userIds and update them in // separate transactions if necessary. - - // ``remainingUsers`` is a placeholder in the event that the number of - // affected users is greater than the maximum number of docs we can update - // in a single transaction. - let remainingUsers: string[] = []; + let usersToUpdate: string[] = []; let totalUsersAssigned = 0; // Run the first transaction to get the user list await db.runTransaction(async (transaction) => { - const usersToUpdate = await getUsersFromOrgs({ + usersToUpdate = await getUsersFromOrgs({ orgs: orgChunk, transaction, includeArchived: false, // Do not assign updated assignment to archived users @@ -226,41 +222,9 @@ export const updateAssignmentsForOrgChunkHandler = async ({ orgChunkSummary: summarizeOrgsForLog(orgChunk), userSummary: summarizeIdListForLog(usersToUpdate), }); - - if (usersToUpdate.length !== 0) { - if (usersToUpdate.length <= MAX_TRANSACTIONS) { - // If the number of users is small enough, update them in this transaction. - if (mode === "update") { - return updateAssignmentForUsers( - usersToUpdate, - administrationId, - administrationData, - transaction - ); - } else { - console.log("adding assignments to users"); - totalUsersAssigned = usersToUpdate.length; - createdUserIds.push(...usersToUpdate); - return addAssignmentToUsers( - usersToUpdate, - administrationId, - administrationData, - transaction - ); - } - } else { - // Otherwise, just save for the next loop over user chunks. - remainingUsers = usersToUpdate; - return Promise.resolve(usersToUpdate.length); - } - } else { - return Promise.resolve(0); - } }); - // If remainingUsersToRemove.length === 0, then these chunks will be of zero length - // and the entire loop below is a no-op. - for (const _userChunk of _chunk(remainingUsers, MAX_TRANSACTIONS)) { + for (const _userChunk of _chunk(usersToUpdate, MAX_TRANSACTIONS)) { await db.runTransaction(async (transaction) => { if (mode === "update") { return updateAssignmentForUsers( From 6b09a21def759537a68c37645b841a137782a639 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Tue, 2 Dec 2025 11:58:56 -0300 Subject: [PATCH 12/15] running transactions in parallel --- .../src/assignments/sync-assignments.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/functions/levante-admin/src/assignments/sync-assignments.ts b/functions/levante-admin/src/assignments/sync-assignments.ts index 762bba0..1ed992c 100644 --- a/functions/levante-admin/src/assignments/sync-assignments.ts +++ b/functions/levante-admin/src/assignments/sync-assignments.ts @@ -224,8 +224,9 @@ export const updateAssignmentsForOrgChunkHandler = async ({ }); }); - for (const _userChunk of _chunk(usersToUpdate, MAX_TRANSACTIONS)) { - await db.runTransaction(async (transaction) => { + const userChunks = _chunk(usersToUpdate, MAX_TRANSACTIONS); + const transactionPromises = userChunks.map((_userChunk) => { + return db.runTransaction(async (transaction) => { if (mode === "update") { return updateAssignmentForUsers( _userChunk, @@ -234,8 +235,6 @@ export const updateAssignmentsForOrgChunkHandler = async ({ transaction ); } else { - totalUsersAssigned += _userChunk.length; - createdUserIds.push(..._userChunk); return addAssignmentToUsers( _userChunk, administrationId, @@ -244,6 +243,13 @@ export const updateAssignmentsForOrgChunkHandler = async ({ ); } }); + }); + + await Promise.all(transactionPromises); + + if (mode === "add") { + totalUsersAssigned = usersToUpdate.length; + createdUserIds.push(...usersToUpdate); } // Update administration stats synchronously for immediate visibility From 2ee7fce31776d105d2267be2255aea5964b81341 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 17 Dec 2025 12:17:30 -0300 Subject: [PATCH 13/15] moved import to the top of the file --- .../levante-admin/src/assignments/assignment-utils.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/functions/levante-admin/src/assignments/assignment-utils.ts b/functions/levante-admin/src/assignments/assignment-utils.ts index eeae81f..c2acd0f 100644 --- a/functions/levante-admin/src/assignments/assignment-utils.ts +++ b/functions/levante-admin/src/assignments/assignment-utils.ts @@ -35,6 +35,7 @@ import { summarizeAssessmentsForLog, summarizeIdListForLog, } from "../utils/logging.js"; +import { updateAdministrationStatsForOrgChunk } from "./on-assignment-updates.js"; /** * Parse a Firestore Timestamp or Date instance @@ -156,7 +157,11 @@ export const rollbackAdministrationCreation = async ( if (creatorDoc.exists) { const fieldPath = new FieldPath("adminData", "administrationsCreated"); - transaction.update(creatorDocRef, fieldPath, FieldValue.arrayRemove(administrationId)); + transaction.update( + creatorDocRef, + fieldPath, + FieldValue.arrayRemove(administrationId) + ); } }); @@ -239,9 +244,6 @@ export const rollbackAssignmentCreation = async ( // Rollback stats if org chunk and administration data are provided if (orgChunk && administrationData && userIds.length > 0) { try { - const { updateAdministrationStatsForOrgChunk } = await import( - "./on-assignment-updates.js" - ); // Decrement stats by the number of users that were rolled back await updateAdministrationStatsForOrgChunk( administrationId, From b1b4da6fa48586e30a83a7cf9834268cce9c9415 Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 17 Dec 2025 12:17:42 -0300 Subject: [PATCH 14/15] removed useless variable --- functions/levante-admin/src/assignments/sync-assignments.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/functions/levante-admin/src/assignments/sync-assignments.ts b/functions/levante-admin/src/assignments/sync-assignments.ts index 1ed992c..83e0118 100644 --- a/functions/levante-admin/src/assignments/sync-assignments.ts +++ b/functions/levante-admin/src/assignments/sync-assignments.ts @@ -208,7 +208,6 @@ export const updateAssignmentsForOrgChunkHandler = async ({ // larger. So we loop through chunks of the userIds and update them in // separate transactions if necessary. let usersToUpdate: string[] = []; - let totalUsersAssigned = 0; // Run the first transaction to get the user list await db.runTransaction(async (transaction) => { @@ -248,18 +247,17 @@ export const updateAssignmentsForOrgChunkHandler = async ({ await Promise.all(transactionPromises); if (mode === "add") { - totalUsersAssigned = usersToUpdate.length; createdUserIds.push(...usersToUpdate); } // Update administration stats synchronously for immediate visibility // Only update stats when adding new assignments (not when updating) - if (mode === "add" && totalUsersAssigned > 0) { + if (mode === "add" && usersToUpdate.length > 0) { await updateAdministrationStatsForOrgChunk( administrationId, orgChunk, administrationData, - totalUsersAssigned + usersToUpdate.length ); } From 7ee83573fc56117b96daf031c1fe6208b4a6f9bd Mon Sep 17 00:00:00 2001 From: Rodrigo Olivares Date: Wed, 17 Dec 2025 14:31:02 -0300 Subject: [PATCH 15/15] small refactoring on parallel functions --- .../levante-admin/src/upsertAdministration.ts | 158 +++++++++++------- 1 file changed, 94 insertions(+), 64 deletions(-) diff --git a/functions/levante-admin/src/upsertAdministration.ts b/functions/levante-admin/src/upsertAdministration.ts index 242884f..16ddcf6 100644 --- a/functions/levante-admin/src/upsertAdministration.ts +++ b/functions/levante-admin/src/upsertAdministration.ts @@ -91,49 +91,63 @@ const createAssignments = async ( const processedChunks: Array<{ orgChunk: IOrgsList; userIds: string[] }> = []; try { - for (let i = 0; i < orgChunks.length; i++) { - const orgChunk = orgChunks[i]; - const result = await updateAssignmentsForOrgChunkHandler({ - administrationId, - administrationData: currData, - orgChunk, - mode: "add", - }); - - if (!result.success) { - // If this chunk failed, rollback all previous chunks - logger.error( - `Assignment creation failed for org chunk ${ - i + 1 - }. Rolling back all previous chunks.`, - { + const chunkResults = await Promise.all( + orgChunks.map(async (orgChunk, i) => { + try { + const result = await updateAssignmentsForOrgChunkHandler({ administrationId, + administrationData: currData, + orgChunk, + mode: "add", + }); + return { orgChunk, result, chunkIndex: i }; + } catch (error: any) { + const err = error instanceof Error ? error : new Error(String(error)); + return { + orgChunk, + result: { success: false, userIds: [], error: err }, chunkIndex: i, - error: result.error, - totalChunksProcessed: processedChunks.length, - } - ); - - // Rollback all previously created assignments - if (allCreatedUserIds.length > 0) { - await rollbackAssignmentCreation(allCreatedUserIds, administrationId); + }; } + }) + ); + + for (const { orgChunk, result } of chunkResults) { + if (result.success) { + allCreatedUserIds.push(...result.userIds); + processedChunks.push({ orgChunk, userIds: result.userIds }); + } + } - // If this is a new administration, also rollback the administration document - if (isNewAdministration && creatorUid) { - await rollbackAdministrationCreation(administrationId, creatorUid); + const failedChunk = chunkResults.find(({ result }) => !result.success); + if (failedChunk) { + logger.error( + `Assignment creation failed for org chunk ${ + failedChunk.chunkIndex + 1 + }. Rolling back all successful chunks.`, + { + administrationId, + chunkIndex: failedChunk.chunkIndex, + error: failedChunk.result.error, + totalChunksProcessed: processedChunks.length, } + ); - throw new Error( - `Failed to create assignment for all participants. ${ - result.error?.message || "Unknown error" - }. Please try again.` - ); + // Rollback all successfully created assignments + if (allCreatedUserIds.length > 0) { + await rollbackAssignmentCreation(allCreatedUserIds, administrationId); + } + + // If this is a new administration, also rollback the administration document + if (isNewAdministration && creatorUid) { + await rollbackAdministrationCreation(administrationId, creatorUid); } - // Track successful chunk - allCreatedUserIds.push(...result.userIds); - processedChunks.push({ orgChunk, userIds: result.userIds }); + throw new Error( + `Failed to create assignment for all participants. ${ + failedChunk.result.error?.message || "Unknown error" + }. Please try again.` + ); } logger.info(`Successfully created assignments for all participants`, { @@ -235,16 +249,18 @@ const updateAssignments = async ( } }); - for (const _userChunk of _chunk(remainingUsersToRemove, MAX_TRANSACTIONS)) { - await db.runTransaction(async (transaction) => { - return removeOrgsFromAssignments( - _userChunk, - [administrationId], - removedExhaustiveOrgs, - transaction - ); - }); - } + await Promise.all( + _chunk(remainingUsersToRemove, MAX_TRANSACTIONS).map((_userChunk) => + db.runTransaction(async (transaction) => { + return removeOrgsFromAssignments( + _userChunk, + [administrationId], + removedExhaustiveOrgs, + transaction + ); + }) + ) + ); } const { minimalOrgs } = await standardizeAdministrationOrgs({ @@ -257,27 +273,41 @@ const updateAssignments = async ( const orgChunks = chunkOrgs(minimalOrgs, 100); - for (let i = 0; i < orgChunks.length; i++) { - const orgChunk = orgChunks[i]; - const result = await updateAssignmentsForOrgChunkHandler({ - administrationId, - administrationData: currData, - orgChunk, - mode: "update", - }); + const chunkResults = await Promise.all( + orgChunks.map(async (orgChunk, i) => { + try { + const result = await updateAssignmentsForOrgChunkHandler({ + administrationId, + administrationData: currData, + orgChunk, + mode: "update", + }); + return { result, chunkIndex: i }; + } catch (error: any) { + const err = error instanceof Error ? error : new Error(String(error)); + return { + result: { success: false, userIds: [], error: err }, + chunkIndex: i, + }; + } + }) + ); - if (!result.success) { - logger.error(`Assignment update failed for org chunk ${i + 1}.`, { + const failedChunk = chunkResults.find(({ result }) => !result.success); + if (failedChunk) { + logger.error( + `Assignment update failed for org chunk ${failedChunk.chunkIndex + 1}.`, + { administrationId, - chunkIndex: i, - error: result.error, - }); - throw new Error( - `Failed to update assignment for all participants. ${ - result.error?.message || "Unknown error" - }. Please try again.` - ); - } + chunkIndex: failedChunk.chunkIndex, + error: failedChunk.result.error, + } + ); + throw new Error( + `Failed to update assignment for all participants. ${ + failedChunk.result.error?.message || "Unknown error" + }. Please try again.` + ); } logger.info(`Successfully updated assignments for all participants`, {