From 86d65e4f3adc84bf512c3a3eec74ded997d68f96 Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 30 Apr 2025 14:51:13 -0800 Subject: [PATCH 1/7] feat: Hyp3 integration for opera rtc --- src/app/models/hyp3/hyp3-job-type.model.ts | 1 + src/app/models/hyp3/hyp3-jobs.model.ts | 28 ++++++++- src/app/services/hyp3/hyp3-api.service.ts | 73 +++++++++++++++++++--- 3 files changed, 93 insertions(+), 9 deletions(-) diff --git a/src/app/models/hyp3/hyp3-job-type.model.ts b/src/app/models/hyp3/hyp3-job-type.model.ts index 531f307b6..6dbb1fa2c 100644 --- a/src/app/models/hyp3/hyp3-job-type.model.ts +++ b/src/app/models/hyp3/hyp3-job-type.model.ts @@ -17,6 +17,7 @@ export interface Hyp3JobProductType { productTypes: string[]; beamModes: string[]; polarizations: string[]; + dateRange?: Range; } export interface Hyp3JobOption { diff --git a/src/app/models/hyp3/hyp3-jobs.model.ts b/src/app/models/hyp3/hyp3-jobs.model.ts index b06bc25ae..88cb1bf05 100644 --- a/src/app/models/hyp3/hyp3-jobs.model.ts +++ b/src/app/models/hyp3/hyp3-jobs.model.ts @@ -278,11 +278,37 @@ export const AutoRift: Hyp3JobType = { options: [] }; +export const OperaRtcJobType: Hyp3JobType = { + id: 'OPERA_RTC', + name: 'OPERA RTC', + // TODO: Update docs url to hyp3 docs + infoUrl: '', + description: 'OPERA_RTC_DESC', + numProducts: 1, + productTypes: [{ + dataset: sentinel_1_bursts, + productTypes: [ + 'BURST', + ], + beamModes: ['IW'], + polarizations: [ + 'VV', 'HH', 'HV', 'VH' + ], + // TODO: Figure out these date ranges + dateRange: { + start: new Date('2016/07/2 00:00:00 UTC'), + end: new Date('2022/01/1 00:00:00 UTC') + }, + }], + options: [] +} + export const hyp3JobTypes = { RTC_GAMMA: RtcGammaJobType, INSAR_GAMMA: InsarGammaJobType, INSAR_ISCE_BURST: InsarIsceBurstJobType, - AUTORIFT: AutoRift + AUTORIFT: AutoRift, + OPERA_RTC: OperaRtcJobType }; export const hyp3JobTypesList = Object.values(hyp3JobTypes); diff --git a/src/app/services/hyp3/hyp3-api.service.ts b/src/app/services/hyp3/hyp3-api.service.ts index e426cc0d1..163c9c56d 100644 --- a/src/app/services/hyp3/hyp3-api.service.ts +++ b/src/app/services/hyp3/hyp3-api.service.ts @@ -223,6 +223,17 @@ export class Hyp3ApiService { hyp3ableProducts.forEach(product => { const prodType = product[0].metadata.productType; + + if (models.OperaRtcJobType.id === jobType.id) { + product = product.map(p => { + if (this.isCrossPolBurst(p)) { + return this.makeCoPolBurst(p); + } else { + return p; + } + }) + } + byProdType[prodType].push(product?.sort((a, b) => { if (a.metadata.date < b.metadata.date) { return -1; @@ -232,9 +243,10 @@ export class Hyp3ApiService { })); }); - const byProductType: models.Hyp3ableByProductType[] = Object.entries(byProdType).map(([productType, prods]) => ({ - productType, products: prods - })); + const byProductType: models.Hyp3ableByProductType[] = Object.entries(byProdType) + .map(([productType, prods]) => { + return ({ productType, products: prods }); + }); return { jobType, @@ -250,6 +262,31 @@ export class Hyp3ApiService { return ({ byJobType, total }); } + private isCrossPolBurst(product: models.CMRProduct) { + return product.metadata.polarization === 'HV' || + product.metadata.polarization === 'VH'; + } + + private makeCoPolBurst(product: models.CMRProduct) { + const crossPol = product.metadata.polarization; + const polarization = crossPol === 'VH' ? 'VV' : 'HH'; + + const name = product.name.replace(`_${crossPol}_`, `_${polarization}_`); + const file = product.file.replace(`_${crossPol}_`, `_${polarization}_`); + const id = product.id.replace(`_${crossPol}_`, `_${polarization}_`); + const downloadUrl = product.downloadUrl.replace(`/${crossPol}/`, `/${polarization}/`); + + const coPolProduct = { + ...product, + name, file, id, downloadUrl, + metadata: { + ...product.metadata, polarization + }, + }; + + return coPolProduct; + } + public getValidJobTypes(product: models.CMRProduct[]): models.Hyp3JobType[] { return models.hyp3JobTypesList.filter(jobType => this.isHyp3able(product, jobType)); } @@ -261,16 +298,36 @@ export class Hyp3ApiService { const types = new Set(productType.productTypes); const pols = new Set(productType.polarizations); const beamModes = new Set(productType.beamModes); - return products.every(product => - types.has(product.metadata.productType) && - pols.has(product.metadata.polarization) && - beamModes.has(product.metadata.beamMode) && - product.dataset !== 'Sentinel-1C' + return products.every(product => { + + return types.has(product.metadata.productType) && + pols.has(product.metadata.polarization) && + beamModes.has(product.metadata.beamMode) && ( + 'dateRange' in productType && + this.withinDateRange(product.metadata.date.toDate(), productType.dateRange) + ) && + product.dataset !== 'Sentinel-1C' + } ); }) ); } + public withinDateRange(check: Date, dateRange: models.DateRange) { + const { start, end } = dateRange; + let isAfterStart = true; + let isBeforeEnd = true; + + if (start !== null) { + isAfterStart = check.getTime() >= start.getTime(); + } + if (end !== null) { + isBeforeEnd = check.getTime() <= end.getTime(); + } + + return isAfterStart && isBeforeEnd; + } + public getExpiredHyp3ableObject(scene: models.CMRProduct): { byJobType: models.Hyp3ableProductByJobType[]; total: number } { const job_types = models.hyp3JobTypes; const job_type = Object.keys(job_types).find(id => { From a0747962fba8a11f4be8157850bccbe849f132e3 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 2 May 2025 07:06:13 -0800 Subject: [PATCH 2/7] chore: change name to OPERA_RTC_S1 and correct date range --- src/app/models/hyp3/hyp3-jobs.model.ts | 15 ++++++++------- src/app/services/hyp3/hyp3-api.service.ts | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/app/models/hyp3/hyp3-jobs.model.ts b/src/app/models/hyp3/hyp3-jobs.model.ts index 88cb1bf05..9e3220d03 100644 --- a/src/app/models/hyp3/hyp3-jobs.model.ts +++ b/src/app/models/hyp3/hyp3-jobs.model.ts @@ -278,12 +278,12 @@ export const AutoRift: Hyp3JobType = { options: [] }; -export const OperaRtcJobType: Hyp3JobType = { - id: 'OPERA_RTC', - name: 'OPERA RTC', +export const OperaRtcS1JobType: Hyp3JobType = { + id: 'OPERA_RTC_S1', + name: 'OPERA RTC S1', // TODO: Update docs url to hyp3 docs infoUrl: '', - description: 'OPERA_RTC_DESC', + description: 'OPERA_RTC_S1_DESC', numProducts: 1, productTypes: [{ dataset: sentinel_1_bursts, @@ -294,9 +294,10 @@ export const OperaRtcJobType: Hyp3JobType = { polarizations: [ 'VV', 'HH', 'HV', 'VH' ], - // TODO: Figure out these date ranges dateRange: { - start: new Date('2016/07/2 00:00:00 UTC'), + // Disallow IPF version < 002.70 according to the dates given at https://sar-mpc.eu/processor/ipf/ + start: new Date('2016/04/13 00:00:00 UTC'), + // Opera RTC forward processing start date end: new Date('2022/01/1 00:00:00 UTC') }, }], @@ -308,7 +309,7 @@ export const hyp3JobTypes = { INSAR_GAMMA: InsarGammaJobType, INSAR_ISCE_BURST: InsarIsceBurstJobType, AUTORIFT: AutoRift, - OPERA_RTC: OperaRtcJobType + OPERA_RTC_S1: OperaRtcS1JobType }; export const hyp3JobTypesList = Object.values(hyp3JobTypes); diff --git a/src/app/services/hyp3/hyp3-api.service.ts b/src/app/services/hyp3/hyp3-api.service.ts index 163c9c56d..a0da9b473 100644 --- a/src/app/services/hyp3/hyp3-api.service.ts +++ b/src/app/services/hyp3/hyp3-api.service.ts @@ -224,7 +224,7 @@ export class Hyp3ApiService { hyp3ableProducts.forEach(product => { const prodType = product[0].metadata.productType; - if (models.OperaRtcJobType.id === jobType.id) { + if (models.OperaRtcS1JobType.id === jobType.id) { product = product.map(p => { if (this.isCrossPolBurst(p)) { return this.makeCoPolBurst(p); From cfea907e92997ae989229ed2fd0e97503798befd Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 2 May 2025 10:05:31 -0800 Subject: [PATCH 3/7] fix: update opera start date --- src/app/models/hyp3/hyp3-jobs.model.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/models/hyp3/hyp3-jobs.model.ts b/src/app/models/hyp3/hyp3-jobs.model.ts index 9e3220d03..b89d13c1a 100644 --- a/src/app/models/hyp3/hyp3-jobs.model.ts +++ b/src/app/models/hyp3/hyp3-jobs.model.ts @@ -296,7 +296,7 @@ export const OperaRtcS1JobType: Hyp3JobType = { ], dateRange: { // Disallow IPF version < 002.70 according to the dates given at https://sar-mpc.eu/processor/ipf/ - start: new Date('2016/04/13 00:00:00 UTC'), + start: new Date('2016/04/14 00:00:00 UTC'), // Opera RTC forward processing start date end: new Date('2022/01/1 00:00:00 UTC') }, From db8ae13a89150d5ffa257adbe2823f17d1a9d97d Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 6 May 2025 13:51:56 -0800 Subject: [PATCH 4/7] Update method for checking duplicates in on demand queue --- src/app/services/hyp3/hyp3-api.service.ts | 7 +++-- src/app/store/queue/queue.reducer.ts | 36 +++++++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/app/services/hyp3/hyp3-api.service.ts b/src/app/services/hyp3/hyp3-api.service.ts index a0da9b473..7829881cf 100644 --- a/src/app/services/hyp3/hyp3-api.service.ts +++ b/src/app/services/hyp3/hyp3-api.service.ts @@ -303,7 +303,6 @@ export class Hyp3ApiService { return types.has(product.metadata.productType) && pols.has(product.metadata.polarization) && beamModes.has(product.metadata.beamMode) && ( - 'dateRange' in productType && this.withinDateRange(product.metadata.date.toDate(), productType.dateRange) ) && product.dataset !== 'Sentinel-1C' @@ -313,7 +312,11 @@ export class Hyp3ApiService { ); } - public withinDateRange(check: Date, dateRange: models.DateRange) { + public withinDateRange(check: Date, dateRange: models.DateRange | null) { + if (!dateRange) { + return true; + } + const { start, end } = dateRange; let isAfterStart = true; let isBeforeEnd = true; diff --git a/src/app/store/queue/queue.reducer.ts b/src/app/store/queue/queue.reducer.ts index f4a348580..73e8447b4 100644 --- a/src/app/store/queue/queue.reducer.ts +++ b/src/app/store/queue/queue.reducer.ts @@ -183,24 +183,34 @@ export function queueReducer(state = initState, action: QueueActions): QueueStat case QueueActionType.ADD_JOBS: { const jobs = [...state.customJobs]; - const new_jobs = [...action.payload]; - let _duplicates = 0; - const jobsToQueue = new_jobs.filter(new_job => - !jobs.some(old_job => { - const result = old_job.job_type === new_job.job_type && - sameGranules(old_job.granules, new_job.granules); - if (result) { - _duplicates += 1; - } - return result; + const jobsToAdd = [...action.payload]; + let duplicates = 0; + + const makeJobId = (job: QueuedHyp3Job) => { + const granuleIds = job.granules.map(g => g.id).join('-'); + return `${job.job_type.id}--${granuleIds}` + } + + const uniqueJobs = new Set(jobs.map(makeJobId)); + + const jobsToQueue = jobsToAdd.filter(jobToAdd => { + const jobId = makeJobId(jobToAdd); + + const isDuplicate = uniqueJobs.has(jobId); + + if (isDuplicate) { + duplicates += 1; + } else { + uniqueJobs.add(jobId); } - ) - ); + + return !isDuplicate; + }); return { ...state, customJobs: [...jobs, ...jobsToQueue], - duplicates: _duplicates + duplicates: duplicates }; } From 6707f46a10cbdc3d72280fc8adf7e71ed2afe3a0 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 6 May 2025 14:01:44 -0800 Subject: [PATCH 5/7] fix: add safe access when getting granule id's --- src/app/store/queue/queue.reducer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/store/queue/queue.reducer.ts b/src/app/store/queue/queue.reducer.ts index 73e8447b4..1c1c1aec7 100644 --- a/src/app/store/queue/queue.reducer.ts +++ b/src/app/store/queue/queue.reducer.ts @@ -187,7 +187,7 @@ export function queueReducer(state = initState, action: QueueActions): QueueStat let duplicates = 0; const makeJobId = (job: QueuedHyp3Job) => { - const granuleIds = job.granules.map(g => g.id).join('-'); + const granuleIds = job.granules.map(g => g?.id).join('-'); return `${job.job_type.id}--${granuleIds}` } From ab2dde16f34a4337a4ae202e6fbcd5d08c046974 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 6 May 2025 17:23:06 -0800 Subject: [PATCH 6/7] chore: small formatting change --- .../on-demand-add-menu.component.ts | 6 +++--- src/app/services/hyp3/hyp3-api.service.ts | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/app/components/shared/on-demand-add-menu/on-demand-add-menu.component.ts b/src/app/components/shared/on-demand-add-menu/on-demand-add-menu.component.ts index 829689263..2dc56a1b0 100644 --- a/src/app/components/shared/on-demand-add-menu/on-demand-add-menu.component.ts +++ b/src/app/components/shared/on-demand-add-menu/on-demand-add-menu.component.ts @@ -96,9 +96,9 @@ export class OnDemandAddMenuComponent implements OnInit { ); } - public queueAllOnDemand(products: models.CMRProduct[][], job_type: models.Hyp3JobType): void { - const jobs: models.QueuedHyp3Job[] = products.map(product => ({ - granules: [...product].sort((a, b) => { + public queueAllOnDemand(productsGroups: models.CMRProduct[][], job_type: models.Hyp3JobType): void { + const jobs: models.QueuedHyp3Job[] = productsGroups.map(products => ({ + granules: [...products].sort((a, b) => { if (a.metadata.date < b.metadata.date) { return -1; } diff --git a/src/app/services/hyp3/hyp3-api.service.ts b/src/app/services/hyp3/hyp3-api.service.ts index 7829881cf..dc5e602c6 100644 --- a/src/app/services/hyp3/hyp3-api.service.ts +++ b/src/app/services/hyp3/hyp3-api.service.ts @@ -298,14 +298,15 @@ export class Hyp3ApiService { const types = new Set(productType.productTypes); const pols = new Set(productType.polarizations); const beamModes = new Set(productType.beamModes); - return products.every(product => { - return types.has(product.metadata.productType) && + return products.every(product => { + return ( + types.has(product.metadata.productType) && pols.has(product.metadata.polarization) && - beamModes.has(product.metadata.beamMode) && ( - this.withinDateRange(product.metadata.date.toDate(), productType.dateRange) - ) && + beamModes.has(product.metadata.beamMode) && + this.withinDateRange(product.metadata.date.toDate(), productType.dateRange) && product.dataset !== 'Sentinel-1C' + ); } ); }) From 68f0e8b5184af330688567fc5b568d51f8e69c78 Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 7 May 2025 13:28:32 -0800 Subject: [PATCH 7/7] Update docs link for opera rtc --- src/app/models/hyp3/hyp3-jobs.model.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/app/models/hyp3/hyp3-jobs.model.ts b/src/app/models/hyp3/hyp3-jobs.model.ts index b89d13c1a..093ae1494 100644 --- a/src/app/models/hyp3/hyp3-jobs.model.ts +++ b/src/app/models/hyp3/hyp3-jobs.model.ts @@ -281,8 +281,7 @@ export const AutoRift: Hyp3JobType = { export const OperaRtcS1JobType: Hyp3JobType = { id: 'OPERA_RTC_S1', name: 'OPERA RTC S1', - // TODO: Update docs url to hyp3 docs - infoUrl: '', + infoUrl: 'https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/', description: 'OPERA_RTC_S1_DESC', numProducts: 1, productTypes: [{