From 01ca149df768a5eb84eed7b620d9866ba5f8e40c Mon Sep 17 00:00:00 2001 From: Roberto Fontanarosa Date: Fri, 18 Jul 2025 23:06:06 +0200 Subject: [PATCH 1/4] improved block to retrieve properties --- functions/src/common/datastore.ts | 13 ++++++ functions/src/export-csv.ts | 74 +++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/functions/src/common/datastore.ts b/functions/src/common/datastore.ts index 9f4b75cc6..2abf86ba6 100644 --- a/functions/src/common/datastore.ts +++ b/functions/src/common/datastore.ts @@ -165,6 +165,19 @@ export class Datastore { .get(); } + fetchPartialLocationsOfInterest( + surveyId: string, + jobId: string, + limit: number + ) { + return this.db_ + .collection(lois(surveyId)) + .where(l.jobId, '==', jobId) + .orderBy(FieldPath.documentId()) + .select('5', '9', '10') + .limit(limit); + } + fetchMailTemplate(templateId: string) { return this.fetchDoc_(mailTemplate(templateId)); } diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts index 28fc1385f..20298101a 100644 --- a/functions/src/export-csv.ts +++ b/functions/src/export-csv.ts @@ -22,7 +22,7 @@ import {getDatastore} from './common/context'; import * as HttpStatus from 'http-status-codes'; import {DecodedIdToken} from 'firebase-admin/auth'; import {List} from 'immutable'; -import {QuerySnapshot} from 'firebase-admin/firestore'; +// import {QuerySnapshot} from 'firebase-admin/firestore'; import {timestampToInt, toMessage} from '@ground/lib'; import {GroundProtos} from '@ground/proto'; import {toGeoJsonGeometry} from '@ground/lib'; @@ -67,8 +67,44 @@ export async function exportCsvHandler( } const {name: jobName} = job; const tasks = job.tasks.sort((a, b) => a.index! - b.index!); - const snapshot = await db.fetchLocationsOfInterest(surveyId, jobId); - const loiProperties = createProperySetFromSnapshot(snapshot); + + // let a = 0; + const loiProperties = new Set(); + let query = db.fetchPartialLocationsOfInterest(surveyId, jobId, 1000); + let lastVisible = null; + do { + const snapshot = await query.get(); + if (snapshot.empty) break; + await Promise.all( + snapshot.docs.map(async doc => { + const loi = doc.data(); + if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return; + Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key)); + }) + ); + // snapshot.forEach(doc => { + // // const loi = toMessage(doc.data(), Pb.LocationOfInterest); + // const loi = doc.data() as { + // 5: string; + // 9: number; + // 10: {[key: string]: any}; + // }; + // // if (loi instanceof Error) return; + // if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return; + // // if (!isAccessibleLoi(loi, ownerId)) return; + // // const properties = loi.properties; + // // for (const key of Object.keys(properties || {})) { + // // loiProperties.add(key); + // // } + // Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key)); + // }); + // a += 1000; + // console.log('Processed: ', a); + lastVisible = snapshot.docs[snapshot.docs.length - 1]; + query = query.startAfter(lastVisible); + } while (lastVisible); + + // const loiProperties = createProperySetFromSnapshot(snapshot); const headers = getHeaders(tasks, loiProperties); res.type('text/csv'); @@ -277,22 +313,22 @@ function getFileName(jobName: string | null) { return `${fileBase}.csv`; } -function createProperySetFromSnapshot( - snapshot: QuerySnapshot, - ownerId?: string -): Set { - const allKeys = new Set(); - snapshot.forEach(doc => { - const loi = toMessage(doc.data(), Pb.LocationOfInterest); - if (loi instanceof Error) return; - if (!isAccessibleLoi(loi, ownerId)) return; - const properties = loi.properties; - for (const key of Object.keys(properties || {})) { - allKeys.add(key); - } - }); - return allKeys; -} +// function createProperySetFromSnapshot( +// snapshot: QuerySnapshot, +// ownerId?: string +// ): Set { +// const allKeys = new Set(); +// snapshot.forEach(doc => { +// const loi = toMessage(doc.data(), Pb.LocationOfInterest); +// if (loi instanceof Error) return; +// if (!isAccessibleLoi(loi, ownerId)) return; +// const properties = loi.properties; +// for (const key of Object.keys(properties || {})) { +// allKeys.add(key); +// } +// }); +// return allKeys; +// } function getPropertiesByName( loi: Pb.LocationOfInterest, From 316c85667a63da597a369ea31f4a588722792de9 Mon Sep 17 00:00:00 2001 From: Roberto Fontanarosa Date: Tue, 2 Sep 2025 12:25:12 +0200 Subject: [PATCH 2/4] fixed wrong condition --- functions/src/export-csv.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts index 20298101a..bd80ab1e5 100644 --- a/functions/src/export-csv.ts +++ b/functions/src/export-csv.ts @@ -78,7 +78,7 @@ export async function exportCsvHandler( await Promise.all( snapshot.docs.map(async doc => { const loi = doc.data(); - if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return; + if (!(ownerId ? loi[9] === 2 && loi[5] === ownerId : true)) return; Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key)); }) ); From 0110fabec5e9cb9dede77c480953a58e46ce9dab Mon Sep 17 00:00:00 2001 From: Roberto Fontanarosa Date: Wed, 3 Sep 2025 10:22:06 +0200 Subject: [PATCH 3/4] removed comments --- functions/src/export-csv.ts | 39 +++---------------------------------- 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts index bd80ab1e5..a0875aaff 100644 --- a/functions/src/export-csv.ts +++ b/functions/src/export-csv.ts @@ -22,7 +22,6 @@ import {getDatastore} from './common/context'; import * as HttpStatus from 'http-status-codes'; import {DecodedIdToken} from 'firebase-admin/auth'; import {List} from 'immutable'; -// import {QuerySnapshot} from 'firebase-admin/firestore'; import {timestampToInt, toMessage} from '@ground/lib'; import {GroundProtos} from '@ground/proto'; import {toGeoJsonGeometry} from '@ground/lib'; @@ -82,24 +81,6 @@ export async function exportCsvHandler( Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key)); }) ); - // snapshot.forEach(doc => { - // // const loi = toMessage(doc.data(), Pb.LocationOfInterest); - // const loi = doc.data() as { - // 5: string; - // 9: number; - // 10: {[key: string]: any}; - // }; - // // if (loi instanceof Error) return; - // if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return; - // // if (!isAccessibleLoi(loi, ownerId)) return; - // // const properties = loi.properties; - // // for (const key of Object.keys(properties || {})) { - // // loiProperties.add(key); - // // } - // Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key)); - // }); - // a += 1000; - // console.log('Processed: ', a); lastVisible = snapshot.docs[snapshot.docs.length - 1]; query = query.startAfter(lastVisible); } while (lastVisible); @@ -313,23 +294,9 @@ function getFileName(jobName: string | null) { return `${fileBase}.csv`; } -// function createProperySetFromSnapshot( -// snapshot: QuerySnapshot, -// ownerId?: string -// ): Set { -// const allKeys = new Set(); -// snapshot.forEach(doc => { -// const loi = toMessage(doc.data(), Pb.LocationOfInterest); -// if (loi instanceof Error) return; -// if (!isAccessibleLoi(loi, ownerId)) return; -// const properties = loi.properties; -// for (const key of Object.keys(properties || {})) { -// allKeys.add(key); -// } -// }); -// return allKeys; -// } - +/** + * Retrieves the values of specified properties from a LocationOfInterest object. + */ function getPropertiesByName( loi: Pb.LocationOfInterest, properties: Set From 12ce6788b7bcfea06bb87ab8f9296a5bf8c5dcd5 Mon Sep 17 00:00:00 2001 From: Roberto Fontanarosa Date: Fri, 21 Nov 2025 18:12:30 +0100 Subject: [PATCH 4/4] improved readibility --- functions/src/export-csv.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts index 95f74adb8..27016d270 100644 --- a/functions/src/export-csv.ts +++ b/functions/src/export-csv.ts @@ -93,8 +93,11 @@ export async function exportCsvHandler( await Promise.all( snapshot.docs.map(async doc => { const loi = doc.data(); - if (!(ownerIdFilter ? loi[9] === 2 && loi[5] === ownerIdFilter : true)) - return; + if (ownerIdFilter) { + if (!(loi[9] === 2 && loi[5] === ownerIdFilter)) { + return; + } + } Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key)); }) );